def package_comments(self, project):
self.logger.info('{} package comments'.format(len(
self.package_results)))
for package, sections in self.package_results.items():
message = 'The version of this package in `{}` has installation issues and may not be installable:'.format(
project)
# Sort sections by text to group binaries together.
sections = sorted(sections, key=lambda s: s.text)
message += '\n\n<pre>\n{}\n</pre>'.format('\n'.join(
[section.text for section in sections]).strip())
# Generate a hash based on the binaries involved and the number of
# sections. This eliminates version or release changes from causing
# an update to the comment while still updating on relevant changes.
binaries = set()
for section in sections:
binaries.update(section.binaries)
info = ';'.join(['::'.join(sorted(binaries)), str(len(sections))])
reference = hashlib.sha1(info).hexdigest()[:7]
# Post comment on devel package in order to notifiy maintainers.
devel_project, devel_package = devel_project_get(
self.apiurl, project, package)
self.comment_write(state='seen',
result=reference,
project=devel_project,
package=devel_package,
message=message)
def origin_devel_project(apiurl, project, package):
if devel_project_simulate.lock:
devel_project, devel_package = devel_project_simulate.lock.get(
apiurl, project, package)
if devel_project:
return devel_project, devel_package
for devel_project, devel_package in origin_devel_project_requests(
apiurl, project, package):
return devel_project, devel_package
return devel_project_get(apiurl, project, package)
def check_action_add_role(self, request, action):
# Decline add_role request (assumed the bot acting on requests to Factory or similar).
message = 'Roles to packages are granted in the devel project, not in %s.' % action.tgt_project
if action.tgt_package is not None:
project, package = devel_project_get(self.apiurl,
action.tgt_project,
action.tgt_package)
message += ' Send this request to {}/{}.'.format(project, package)
self.review_messages['declined'] = message
return False
def origin_maintainer_review_ensure(self, origin_info, package, message, request=None):
if not request:
request = self.request
origin = origin_workaround_strip(origin_info.project)
# Check if the origin package is actually developed elsewhere and add
# the maintainer review for the development location.
devel_project, _ = devel_project_get(self.apiurl, origin, package)
if devel_project:
origin = devel_project
users = package_role_expand(self.apiurl, origin, package, 'maintainer')
if request.creator not in users:
self.add_review(request, by_project=origin, by_package=package, msg=message)
def config_origin_generator(origins, apiurl=None, project=None, package=None, skip_workarounds=False):
for origin_item in origins:
for origin, values in origin_item.items():
is_workaround = origin_workaround_check(origin)
if skip_workarounds and is_workaround:
break
if (origin == '<devel>' or origin == '<devel>~') and apiurl and project and package:
devel_project, devel_package = devel_project_get(apiurl, project, package)
if not devel_project:
break
origin = devel_project
if is_workaround:
origin = origin_workaround_ensure(origin)
yield origin, values
break # Only support single value inside list item.
def config_origin_generator(origins, apiurl=None, project=None, package=None, skip_workarounds=False):
for origin_item in origins:
for origin, values in origin_item.items():
is_workaround = origin_workaround_check(origin)
if skip_workarounds and is_workaround:
break
if (origin == '<devel>' or origin == '<devel>~') and apiurl and project and package:
devel_project, devel_package = devel_project_get(apiurl, project, package)
if not devel_project:
break
origin = devel_project
if is_workaround:
origin = origin_workaround_ensure(origin)
yield origin, values
break # Only support single value inside list item.
def check_source_submission_inner(self, src_project, src_package, src_rev, target_project, target_package):
super(Leaper, self).check_source_submission(src_project, src_package, src_rev, target_project, target_package)
self.automatic_submission = False
if src_project == target_project and src_package == target_package:
self.logger.info('self submission detected')
self.needs_release_manager = True
return True
src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev)
package = target_package
origin = self.lookup.get(target_project, package)
origin_same = True
if origin:
origin_same = self._check_same_origin(origin, src_project)
if src_srcinfo is None:
# source package does not exist?
# handle here to avoid crashing on the next line
self.logger.warn("Could not get source info for %s/%s@%s" % (src_project, src_package, src_rev))
return False
if self.ibs and target_project.startswith('SUSE:SLE'):
review_result = None
prj = 'openSUSE.org:openSUSE:Factory'
# True or None (open request) are acceptable for SLE.
in_factory = self._check_factory(package, src_srcinfo, prj)
if in_factory:
review_result = True
self.source_in_factory = True
elif in_factory is None:
self.pending_factory_submission = True
else:
if not self.is_package_in_project(prj, package):
self.logger.info('the package is not in Factory, nor submitted there')
else:
self.logger.info('different sources in {}'.format(self.rdiff_link(src_project, src_package, src_rev, prj, package)))
if review_result == None and not self.pending_factory_submission:
other_projects_to_check = []
m = re.match(r'SUSE:SLE-(\d+)(?:-SP(\d+)):', target_project)
if m:
sle_version = int(m.group(1))
sp_version = int(m.group(2))
versions_to_check = []
# yeah, too much harcoding here
if sle_version == 12:
versions_to_check = [ '42.3' ]
elif sle_version == 15:
versions_to_check = [ '15.%d'%i for i in range(sp_version+1) ]
else:
self.logger.error("can't handle %d.%d", sle_version, sp_version)
for version in versions_to_check:
leap = 'openSUSE.org:openSUSE:Leap:%s'%(version)
other_projects_to_check += [ leap, leap + ':Update', leap + ':NonFree', leap + ':NonFree:Update' ]
for prj in other_projects_to_check:
if self.is_package_in_project(prj, package):
self.logger.debug('checking {}'.format(prj))
if self._check_factory(package, src_srcinfo, prj) is True:
self.logger.info('found source match in {}'.format(prj))
devel_project, devel_package = devel_project_get(self.apiurl, 'openSUSE.org:openSUSE:Factory', package)
if devel_project is not None:
# specifying devel package is optional
if devel_package is None:
devel_package = package
if self.is_package_in_project(devel_project, devel_package):
if self._check_matching_srcmd5(devel_project, devel_package, src_srcinfo.verifymd5) == True:
self.logger.info('matching sources in {}/{}'.format(devel_project, devel_package))
return True
else:
self.logger.info('different sources in devel project {}'.format(self.rdiff_link(src_project, src_package, src_rev, devel_project, devel_package)))
else:
self.logger.info('no devel project found for {}/{}'.format('openSUSE.org:openSUSE:Factory', package))
self.logger.info('no matching sources found anywhere. Needs a human to decide whether that is ok. Please provide some justification to help that person.')
else:
leap = 'openSUSE.org:openSUSE:Leap:15.1'
if not self.is_package_in_project(target_project, package) \
and self.is_package_in_project(leap, package) \
and self._check_factory(package, src_srcinfo, leap) is False:
self.logger.info('different sources in {}'.format(self.rdiff_link(src_project, src_package, src_rev, leap, package)))
if not review_result and origin is not None:
review_result = origin_same
if origin_same or origin == 'openSUSE.org:openSUSE:Factory' and self.pending_factory_submission:
self.logger.info("ok, origin %s unchanged", origin)
else:
# only log origin state if it's taken into consideration for the review result
self.logger.info("Submitted from a different origin than expected ('%s')", origin)
self.needs_release_manager = True
# no result so far and also no factory submission to wait
# for. So just pass to avoid requring too much overrides
if not self.pending_factory_submission:
review_result = True
if not review_result and self.override_allow:
# Rather than decline, leave review open in-case of change and
# ask release manager for input via override comment.
self.logger.info('Comment `(at){} override accept` to force accept.'.format(self.review_user))
self.needs_release_manager = True
review_result = None
return review_result
if target_project.endswith(':Update'):
self.logger.info("expected origin is '%s' (%s)", origin,
"unchanged" if origin_same else "changed")
# Only when not from current product should request require maintainer review.
self.do_check_maintainer_review = False
if origin_same:
return True
good = self._check_matching_srcmd5(origin, target_package, src_srcinfo.verifymd5)
if good:
self.logger.info('submission source found in origin ({})'.format(origin))
return good
good = self.factory._check_requests(origin, target_package, src_srcinfo.verifymd5)
if good or good == None:
self.logger.info('found pending submission against origin ({})'.format(origin))
return good
# TODO #1662: Uncomment once maintbot has been superseded and leaper
# is no longer run in comment-only mode.
#self.do_check_maintainer_review = True
return None
elif self.action.type == 'maintenance_incident':
self.logger.debug('unhandled incident pattern (targetting non :Update project)')
return True
# obviously
if src_project in ('openSUSE:Factory', 'openSUSE:Factory:NonFree'):
self.source_in_factory = True
is_fine_if_factory = False
not_in_factory_okish = False
if origin:
self.logger.info("expected origin is '%s' (%s)", origin,
"unchanged" if origin_same else "changed")
if origin.startswith('Devel;'):
if origin_same == False:
self.logger.debug("not submitted from devel project")
return False
is_fine_if_factory = True
not_in_factory_okish = True
if self.must_approve_version_updates:
self.needs_release_manager = True
# fall through to check history and requests
elif origin.startswith('openSUSE:Factory'):
# A large number of requests are created by hand that leaper
# would have created via update_crawler.py. This applies to
# other origins, but primary looking to let Factory submitters
# know that there is no need to make manual submissions to both.
# Since it has a lookup entry it is not a new package.
self.automatic_submission = False
if self.must_approve_version_updates:
self.needs_release_manager = True
if origin == src_project:
self.source_in_factory = True
# no need to approve submissions from Factory if
# the lookup file points to Factory. Just causes
# spam for many maintainers #1393
self.do_check_maintainer_review = False
is_fine_if_factory = True
# fall through to check history and requests
elif origin == 'FORK':
is_fine_if_factory = True
if not src_project.startswith('SUSE:SLE-'):
not_in_factory_okish = True
self.needs_check_source = True
self.needs_release_manager = True
# fall through to check history and requests
# TODO Ugly save for 15.1 (n-1).
elif origin.startswith('openSUSE:Leap:15.0'):
if self.must_approve_maintenance_updates:
self.needs_release_manager = True
if src_project.startswith('openSUSE:Leap'):
self.do_check_maintainer_review = False
# submitted from :Update
if origin_same:
self.logger.debug("submission from 15.0 ok")
return True
# switching to sle package might make sense
if src_project.startswith('SUSE:SLE-15'):
self.needs_release_manager = True
self.do_check_maintainer_review = False
return True
# submitted from elsewhere but is in :Update
else:
good = self._check_matching_srcmd5('openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5)
if good:
self.logger.info("submission found in 15.0")
return good
# check release requests too
good = self.factory._check_requests('openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5)
if good or good == None:
self.logger.debug("found request")
return good
# let's see where it came from before
oldorigin = self.lookup.get('openSUSE:Leap:15.0', target_package)
if oldorigin:
self.logger.debug("oldorigin {}".format(oldorigin))
# Factory. So it's ok to keep upgrading it to Factory
# TODO: whitelist packages where this is ok and block others?
self.logger.info("Package was from %s in 15.0", oldorigin)
if oldorigin.startswith('openSUSE:Factory'):
# check if an attempt to switch to SLE package is made
for sp in ('SP1:GA', 'SP1:Update'):
good = self._check_matching_srcmd5('SUSE:SLE-15-{}'.format(sp), target_package, src_srcinfo.verifymd5)
if good:
self.logger.info("request sources come from SLE")
self.needs_release_manager = True
return good
# TODO Ugly save for 15.2 (n-2).
elif False and oldorigin.startswith('openSUSE:Leap:15.0'):
self.logger.info("Package was from %s in 15.0", oldorigin)
# the release manager needs to review attempts to upgrade to Factory
is_fine_if_factory = True
self.needs_release_manager = True
elif origin.startswith('SUSE:SLE-15'):
if self.must_approve_maintenance_updates:
self.needs_release_manager = True
if src_project.startswith('SUSE:SLE-15'):
self.do_check_maintainer_review = False
for v in ('15.0', '15.1'):
prj = 'openSUSE:Leap:{}:SLE-workarounds'.format(v)
if self.is_package_in_project( prj, target_package):
self.logger.info("found package in %s", prj)
if not self._check_matching_srcmd5(prj,
target_package,
src_srcinfo.verifymd5):
self.logger.info("sources in %s are NOT identical",
self.rdiff_link(src_project, src_package, src_rev, prj, package))
self.needs_release_manager = True
# submitted from :Update
if origin == src_project:
self.logger.debug("submission origin ok")
return True
elif origin.endswith(':GA') \
and src_project == origin[:-2]+'Update':
self.logger.debug("sle update submission")
return True
# check if submitted from higher SP
priolist = ['SUSE:SLE-15:', 'SUSE:SLE-15-SP1:', 'SUSE:SLE-15-SP2:', 'SUSE:SLE-15-SP3:']
for i in range(len(priolist)-1):
if origin.startswith(priolist[i]):
for prj in priolist[i+1:]:
if src_project.startswith(prj):
self.logger.info("submission from higher service pack %s:* ok", prj)
return True
in_sle_origin = self._check_factory(target_package, src_srcinfo, origin)
if in_sle_origin:
self.logger.info('parallel submission, also in {}'.format(origin))
return True
self.needs_release_manager = True
# the release manager needs to review attempts to upgrade to Factory
is_fine_if_factory = True
else:
self.logger.error("unhandled origin %s", origin)
return False
else: # no origin
# submission from SLE is ok
if src_project.startswith('SUSE:SLE-15'):
self.do_check_maintainer_review = False
return True
# new package submitted from Factory. Check if it was in
# 42.3 before and skip maintainer review if so.
subprj = src_project[len('openSUSE:Factory'):]
# disabled for reference. Needed again for 16.0 probably
if False and self.source_in_factory and target_project.startswith('openSUSE:Leap:15.0') \
and self.is_package_in_project('openSUSE:Leap:42.3'+subprj, package):
self.logger.info('package was in 42.3')
self.do_check_maintainer_review = False
return True
is_fine_if_factory = True
self.needs_release_manager = True
if origin is None or not origin.startswith('SUSE:SLE-'):
for p in (':Update', ':GA'):
prj = 'SUSE:SLE-15' + p
if self.is_package_in_project(prj, package):
self.logger.info('Package is in {}'.format(
self.rdiff_link(src_project, src_package, src_rev, prj, package)))
break
is_in_factory = self.source_in_factory
# we came here because none of the above checks find it good, so
# let's see if the package is in Factory at least
if is_in_factory is None:
is_in_factory = self._check_factory(package, src_srcinfo)
if is_in_factory:
self.source_in_factory = True
self.needs_reviewteam = False
self.needs_legal_review = False
elif is_in_factory is None:
self.pending_factory_submission = True
self.needs_reviewteam = False
self.needs_legal_review = False
else:
if src_project.startswith('SUSE:SLE-15') \
or src_project.startswith('openSUSE:Leap:15.'):
self.needs_reviewteam = False
self.needs_legal_review = False
else:
self.needs_reviewteam = True
self.needs_legal_review = True
self.source_in_factory = False
if is_fine_if_factory:
if self.source_in_factory:
return True
elif self.pending_factory_submission:
return None
elif not_in_factory_okish:
self.needs_reviewteam = True
self.needs_legal_review = True
return True
if self.override_allow:
# Rather than decline, leave review open and ask release
# manager for input via override comment.
self.logger.info('Comment `(at){} override accept` to force accept.'.format(self.review_user))
self.needs_release_manager = True
return None
return False
def update_targets(self, targets, sources):
# special case maintenance project. Only consider main
# package names. The code later follows the link in the
# source project then.
if self.is_maintenance_project(self.from_prj):
mainpacks = set()
for package, sourceinfo in sources.items():
if package.startswith('patchinfo.'):
continue
files = set([node.text for node in sourceinfo.findall('filename')])
if '{}.spec'.format(package) in files:
mainpacks.add(package)
sources = { package: sourceinfo for package, sourceinfo in sources.iteritems() if package in mainpacks }
for package, sourceinfo in sources.items():
origin = self.lookup.get(package, '')
if origin.startswith('Devel;'):
(dummy, origin, dummy) = origin.split(';')
if self.filter_lookup and not origin in self.filter_lookup:
if not origin.startswith('subpackage of'):
self.skipped.setdefault(origin, set()).add(package)
continue
if not package in targets:
if not self.submit_new:
logging.info('Package %s not found in targets' % (package))
continue
if self.is_source_innerlink(self.from_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
else:
targetinfo = targets[package]
# XXX: make more generic :-)
devel_prj = devel_project_get(self.apiurl, FACTORY, package)
if devel_prj == 'devel:languages:haskell':
logging.info('skipping haskell package %s' % package)
continue
# Compare verifymd5
md5_from = sourceinfo.get('verifymd5')
md5_to = targetinfo.get('verifymd5')
if md5_from == md5_to:
#logging.info('Package %s not marked for update' % package)
continue
if self.is_source_innerlink(self.to_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
# this makes only sense if we look at the expanded view
# and want to submit from proper project
# originproject = default_origin
# if not sourceinfo.find('originproject') is None:
# originproject = sourceinfo.find('originproject').text
# logging.warn('changed originproject for {} to {}'.format(package, originproject))
src_project, src_package, src_rev = self.follow_link(self.from_prj, package,
sourceinfo.get('srcmd5'),
sourceinfo.get('verifymd5'))
res = self.submitrequest(src_project, src_package, src_rev, package, origin)
if res:
logging.info('Created request %s for %s' % (res, package))
elif res != 0:
logging.error('Error creating the request for %s' % package)
def crawl(self):
"""Main method"""
succeeded_packages = []
succeeded_packages = self.get_build_succeeded_packages(self.from_prj)
if not len(succeeded_packages) > 0:
logging.info('No build succeeded package in %s' % self.from_prj)
return
# randomize the list
random.shuffle(succeeded_packages)
# get souce packages from target
target_packages = self.get_source_packages(self.to_prj)
deleted_packages = self.get_deleted_packages(self.to_prj)
if self.to_prj.startswith("openSUSE:"):
for prd in OPENSUSE_RELEASED_VERSION:
deleted_packages = deleted_packages + self.get_deleted_packages(prd)
pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, 'openSUSE:Factory')
skip_pkgs_list = self.load_skip_pkgs_list(pseudometa_project, pseudometa_package).splitlines()
ms_packages = [] # collect multi specs packages
for i in range(0, min(int(self.submit_limit), len(succeeded_packages))):
package = succeeded_packages[i]
submit_ok = True
if package in deleted_packages:
logging.info('%s has been dropped from %s, ignore it!' % (package, self.to_prj))
submit_ok = False
if self.is_sle_base_pkgs(package) is True:
logging.info('%s origin from SLE base, skip for now!' % package)
submit_ok = False
# make sure it is new package
new_pkg = self.is_new_package(self.to_prj, package)
if new_pkg is not True:
logging.info('%s is not a new package, do not submit.' % package)
submit_ok = False
multi_specs = self.check_multiple_specfiles(self.factory, package)
if multi_specs is None:
logging.info('%s does not exist in %s' % (package, 'openSUSE:Factory'))
submit_ok = False
if multi_specs:
if multi_specs['linkinfo']:
logging.info('%s in %s is sub-package of %s, skip it!' %
(package, 'openSUSE:Factory', multi_specs['linkinfo']))
ms_packages.append(package)
submit_ok = False
for spec in multi_specs['specs']:
if spec not in succeeded_packages:
logging.info('%s is sub-pacakge of %s but build failed, skip it!' % (spec, package))
submit_ok = False
if not submit_ok:
continue
# make sure the package non-exist in target yet ie. expand=False
if package not in target_packages:
# make sure there is no request against same package
request = self.get_request_list(package)
if request:
logging.debug("There is a request to %s / %s already or it has been declined/revoked, skip!" %
(package, self.to_prj))
else:
logging.info("%d - Preparing submit %s to %s" % (i, package, self.to_prj))
# get devel project
devel_prj, devel_pkg = devel_project_get(self.apiurl, self.factory, package)
# check devel project does not in the skip list
if devel_prj in self.skip_devel_project_list:
# check the except packages list
match = None
for elem in self.except_pkgs_list:
m = re.search(elem, package)
if m is not None:
match = True
if match is not True:
logging.info('%s/%s is in the skip list, do not submit.' % (devel_prj, package))
continue
else:
pass
# check package does not in the skip list
match = None
for elem in skip_pkgs_list:
m = re.search(str(elem), package)
if m is not None:
match = True
if match is True:
logging.info('%s is in the skip list, do not submit.' % package)
continue
else:
pass
res = self.create_submitrequest(package)
if res and res is not None:
logging.info('Created request %s for %s' % (res, package))
else:
logging.error('Error occurred when creating submit request')
else:
logging.debug('%s is exist in %s, skip!' % (package, self.to_prj))
time.sleep(5)
# dump multi specs packages
print("Multi-specfile packages:")
if ms_packages:
for pkg in ms_packages:
print(pkg)
else:
print('None')
def update_targets(self, targets, sources):
# special case maintenance project. Only consider main
# package names. The code later follows the link in the
# source project then.
if self.is_maintenance_project(self.from_prj):
mainpacks = set()
for package, sourceinfo in sources.items():
if package.startswith('patchinfo.'):
continue
files = set(
[node.text for node in sourceinfo.findall('filename')])
if '{}.spec'.format(package) in files:
mainpacks.add(package)
sources = {
package: sourceinfo
for package, sourceinfo in sources.iteritems()
if package in mainpacks
}
for package, sourceinfo in sources.items():
origin = self.lookup.get(package, '')
if origin.startswith('Devel;'):
(dummy, origin, dummy) = origin.split(';')
if self.filter_lookup and not origin in self.filter_lookup:
if not origin.startswith('subpackage of'):
self.skipped.setdefault(origin, set()).add(package)
continue
if not package in targets:
if not self.submit_new:
logging.info('Package %s not found in targets' % (package))
continue
if self.is_source_innerlink(self.from_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
else:
targetinfo = targets[package]
# XXX: make more generic :-)
devel_prj = devel_project_get(self.apiurl, FACTORY, package)
if devel_prj == 'devel:languages:haskell':
logging.info('skipping haskell package %s' % package)
continue
# Compare verifymd5
md5_from = sourceinfo.get('verifymd5')
md5_to = targetinfo.get('verifymd5')
if md5_from == md5_to:
#logging.info('Package %s not marked for update' % package)
continue
if self.is_source_innerlink(self.to_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
# this makes only sense if we look at the expanded view
# and want to submit from proper project
# originproject = default_origin
# if not sourceinfo.find('originproject') is None:
# originproject = sourceinfo.find('originproject').text
# logging.warn('changed originproject for {} to {}'.format(package, originproject))
src_project, src_package, src_rev = self.follow_link(
self.from_prj, package, sourceinfo.get('srcmd5'),
sourceinfo.get('verifymd5'))
res = self.submitrequest(src_project, src_package, src_rev,
package, origin)
if res:
logging.info('Created request %s for %s' % (res, package))
elif res != 0:
logging.error('Error creating the request for %s' % package)
def check_source_submission_inner(self, src_project, src_package, src_rev,
target_project, target_package):
super(Leaper,
self).check_source_submission(src_project, src_package, src_rev,
target_project, target_package)
self.automatic_submission = False
if src_project == target_project and src_package == target_package:
self.logger.info('self submission detected')
self.needs_release_manager = True
return True
src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev)
package = target_package
origin = self.lookup.get(target_project, package)
origin_same = True
if origin:
origin_same = self._check_same_origin(origin, src_project)
if src_srcinfo is None:
# source package does not exist?
# handle here to avoid crashing on the next line
self.logger.warn("Could not get source info for %s/%s@%s" %
(src_project, src_package, src_rev))
return False
if self.ibs and target_project.startswith('SUSE:SLE'):
review_result = None
prj = 'openSUSE.org:openSUSE:Factory'
if self.is_package_in_project(prj, package):
# True or None (open request) are acceptable for SLE.
in_factory = self._check_factory(package, src_srcinfo, prj)
if in_factory:
review_result = True
self.source_in_factory = True
elif in_factory is None:
self.pending_factory_submission = True
else:
self.logger.info('different sources in {}'.format(
self.rdiff_link(src_project, src_package, src_rev, prj,
package)))
else:
self.logger.info(
'the package is not in Factory, nor submitted there')
if review_result == None:
other_projects_to_check = []
m = re.match('SUSE:SLE-(\d+)(?:-SP(\d+)):', target_project)
if m:
sle_version = int(m.group(1))
sp_version = int(m.group(2))
versions_to_check = []
# yeah, too much harcoding here
if sle_version == 12:
versions_to_check = ['42.3']
elif sle_version == 15:
versions_to_check = [
'15.%d' % i for i in range(sp_version + 1)
]
else:
self.logger.error("can't handle %d.%d", sle_version,
sp_version)
for version in versions_to_check:
leap = 'openSUSE.org:openSUSE:Leap:%s' % (version)
other_projects_to_check += [
leap, leap + ':Update', leap + ':NonFree',
leap + ':NonFree:Update'
]
for prj in other_projects_to_check:
if self.is_package_in_project(prj, package):
self.logger.info('checking {}'.format(prj))
if self._check_factory(package, src_srcinfo,
prj) is True:
self.logger.info(
'found source match in {}'.format(prj))
else:
self.logger.info('different sources in {}'.format(
self.rdiff_link(src_project, src_package,
src_rev, prj, package)))
devel_project, devel_package = devel_project_get(
self.apiurl, 'openSUSE.org:openSUSE:Factory', package)
if devel_project is not None:
# specifying devel package is optional
if devel_package is None:
devel_package = package
if self.is_package_in_project(devel_project,
devel_package):
if self._check_matching_srcmd5(
devel_project, devel_package,
src_srcinfo.verifymd5) == True:
self.logger.info(
'matching sources in {}/{}'.format(
devel_project, devel_package))
return True
else:
self.logger.info(
'different sources in devel project {}'.format(
self.rdiff_link(src_project, src_package,
src_rev, devel_project,
devel_package)))
else:
self.logger.info('no devel project found for {}/{}'.format(
'openSUSE.org:openSUSE:Factory', package))
self.logger.info(
'no matching sources found anywhere. Needs a human to decide whether that is ok. Please provide some justification to help that person.'
)
if not review_result:
review_result = origin_same
if origin_same:
self.logger.info("ok, origin %s unchanged", origin)
else:
# only log origin state if it's taken into consideration for the review result
self.logger.info(
"Submitted from a different origin than expected ('%s')",
origin)
if not review_result and self.override_allow:
# Rather than decline, leave review open in-case of change and
# ask release manager for input via override comment.
self.logger.info(
'Comment `(at){} override accept` to force accept.'.format(
self.review_user))
self.needs_release_manager = True
review_result = None
return review_result
if target_project.endswith(':Update'):
self.logger.info("expected origin is '%s' (%s)", origin,
"unchanged" if origin_same else "changed")
# Only when not from current product should request require maintainer review.
self.do_check_maintainer_review = False
if origin_same:
return True
good = self._check_matching_srcmd5(origin, target_package,
src_srcinfo.verifymd5)
if good:
self.logger.info(
'submission source found in origin ({})'.format(origin))
return good
good = self.factory._check_requests(origin, target_package,
src_srcinfo.verifymd5)
if good or good == None:
self.logger.info(
'found pending submission against origin ({})'.format(
origin))
return good
# TODO #1662: Uncomment once maintbot has been superseded and leaper
# is no longer run in comment-only mode.
#self.do_check_maintainer_review = True
return None
elif self.action.type == 'maintenance_incident':
self.logger.debug(
'unhandled incident pattern (targetting non :Update project)')
return True
# obviously
if src_project in ('openSUSE:Factory', 'openSUSE:Factory:NonFree'):
self.source_in_factory = True
is_fine_if_factory = False
not_in_factory_okish = False
if origin:
self.logger.info("expected origin is '%s' (%s)", origin,
"unchanged" if origin_same else "changed")
if origin.startswith('Devel;'):
if origin_same == False:
self.logger.debug("not submitted from devel project")
return False
is_fine_if_factory = True
not_in_factory_okish = True
if self.must_approve_version_updates:
self.needs_release_manager = True
# fall through to check history and requests
elif origin.startswith('openSUSE:Factory'):
# A large number of requests are created by hand that leaper
# would have created via update_crawler.py. This applies to
# other origins, but primary looking to let Factory submitters
# know that there is no need to make manual submissions to both.
# Since it has a lookup entry it is not a new package.
self.automatic_submission = False
if self.must_approve_version_updates:
self.needs_release_manager = True
if origin == src_project:
self.source_in_factory = True
# no need to approve submissions from Factory if
# the lookup file points to Factory. Just causes
# spam for many maintainers #1393
self.do_check_maintainer_review = False
is_fine_if_factory = True
# fall through to check history and requests
elif origin == 'FORK':
is_fine_if_factory = True
if not src_project.startswith('SUSE:SLE-'):
not_in_factory_okish = True
self.needs_check_source = True
self.needs_release_manager = True
# fall through to check history and requests
# TODO Ugly save for 15.1 (n-1).
elif origin.startswith('openSUSE:Leap:15.0'):
if self.must_approve_maintenance_updates:
self.needs_release_manager = True
if src_project.startswith('openSUSE:Leap'):
self.do_check_maintainer_review = False
# submitted from :Update
if origin_same:
self.logger.debug("submission from 15.0 ok")
return True
# switching to sle package might make sense
if src_project.startswith('SUSE:SLE-15'):
self.needs_release_manager = True
self.do_check_maintainer_review = False
return True
# submitted from elsewhere but is in :Update
else:
good = self._check_matching_srcmd5(
'openSUSE:Leap:15.0:Update', target_package,
src_srcinfo.verifymd5)
if good:
self.logger.info("submission found in 15.0")
return good
# check release requests too
good = self.factory._check_requests(
'openSUSE:Leap:15.0:Update', target_package,
src_srcinfo.verifymd5)
if good or good == None:
self.logger.debug("found request")
return good
# let's see where it came from before
oldorigin = self.lookup.get('openSUSE:Leap:15.0',
target_package)
if oldorigin:
self.logger.debug("oldorigin {}".format(oldorigin))
# Factory. So it's ok to keep upgrading it to Factory
# TODO: whitelist packages where this is ok and block others?
self.logger.info("Package was from %s in 15.0", oldorigin)
if oldorigin.startswith('openSUSE:Factory'):
# check if an attempt to switch to SLE package is made
for sp in ('SP1:GA', 'SP1:Update'):
good = self._check_matching_srcmd5(
'SUSE:SLE-15-{}'.format(sp), target_package,
src_srcinfo.verifymd5)
if good:
self.logger.info(
"request sources come from SLE")
self.needs_release_manager = True
return good
# TODO Ugly save for 15.2 (n-2).
elif False and oldorigin.startswith('openSUSE:Leap:15.0'):
self.logger.info("Package was from %s in 15.0",
oldorigin)
# the release manager needs to review attempts to upgrade to Factory
is_fine_if_factory = True
self.needs_release_manager = True
elif origin.startswith('SUSE:SLE-15'):
if self.must_approve_maintenance_updates:
self.needs_release_manager = True
if src_project.startswith('SUSE:SLE-15'):
self.do_check_maintainer_review = False
for v in ('15.0', '15.1'):
prj = 'openSUSE:Leap:{}:SLE-workarounds'.format(v)
if self.is_package_in_project(prj, target_package):
self.logger.info("found package in %s", prj)
if not self._check_matching_srcmd5(
prj, target_package, src_srcinfo.verifymd5):
self.logger.info(
"sources in %s are NOT identical",
self.rdiff_link(src_project, src_package,
src_rev, prj, package))
self.needs_release_manager = True
# submitted from :Update
if origin == src_project:
self.logger.debug("submission origin ok")
return True
elif origin.endswith(':GA') \
and src_project == origin[:-2]+'Update':
self.logger.debug("sle update submission")
return True
# check if submitted from higher SP
priolist = [
'SUSE:SLE-15:', 'SUSE:SLE-15-SP1:', 'SUSE:SLE-15-SP2:',
'SUSE:SLE-15-SP3:'
]
for i in range(len(priolist) - 1):
if origin.startswith(priolist[i]):
for prj in priolist[i + 1:]:
if src_project.startswith(prj):
self.logger.info(
"submission from higher service pack %s:* ok",
prj)
return True
in_sle_origin = self._check_factory(target_package,
src_srcinfo, origin)
if in_sle_origin:
self.logger.info(
'parallel submission, also in {}'.format(origin))
return True
self.needs_release_manager = True
# the release manager needs to review attempts to upgrade to Factory
is_fine_if_factory = True
else:
self.logger.error("unhandled origin %s", origin)
return False
else: # no origin
# submission from SLE is ok
if src_project.startswith('SUSE:SLE-15'):
self.do_check_maintainer_review = False
return True
# new package submitted from Factory. Check if it was in
# 42.3 before and skip maintainer review if so.
subprj = src_project[len('openSUSE:Factory'):]
# disabled for reference. Needed again for 16.0 probably
if False and self.source_in_factory and target_project.startswith('openSUSE:Leap:15.0') \
and self.is_package_in_project('openSUSE:Leap:42.3'+subprj, package):
self.logger.info('package was in 42.3')
self.do_check_maintainer_review = False
return True
is_fine_if_factory = True
self.needs_release_manager = True
if origin is None or not origin.startswith('SUSE:SLE-'):
for p in (':Update', ':GA'):
prj = 'SUSE:SLE-15' + p
if self.is_package_in_project(prj, package):
self.logger.info('Package is in {}'.format(
self.rdiff_link(src_project, src_package, src_rev, prj,
package)))
break
is_in_factory = self.source_in_factory
# we came here because none of the above checks find it good, so
# let's see if the package is in Factory at least
if is_in_factory is None:
is_in_factory = self._check_factory(package, src_srcinfo)
if is_in_factory:
self.source_in_factory = True
self.needs_reviewteam = False
self.needs_legal_review = False
elif is_in_factory is None:
self.pending_factory_submission = True
self.needs_reviewteam = False
self.needs_legal_review = False
else:
if src_project.startswith('SUSE:SLE-15') \
or src_project.startswith('openSUSE:Leap:15.'):
self.needs_reviewteam = False
self.needs_legal_review = False
else:
self.needs_reviewteam = True
self.needs_legal_review = True
self.source_in_factory = False
if is_fine_if_factory:
if self.source_in_factory:
return True
elif self.pending_factory_submission:
return None
elif not_in_factory_okish:
self.needs_reviewteam = True
self.needs_legal_review = True
return True
if self.override_allow:
# Rather than decline, leave review open and ask release
# manager for input via override comment.
self.logger.info(
'Comment `(at){} override accept` to force accept.'.format(
self.review_user))
self.needs_release_manager = True
return None
return False
def check_source_submission(self, src_project, src_package, src_rev,
target_project, target_package):
super(Leaper,
self).check_source_submission(src_project, src_package, src_rev,
target_project, target_package)
self.automatic_submission = False
if src_project == target_project and src_package == target_package:
self.logger.info('self submission detected')
self.needs_release_manager = True
return True
src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev)
package = target_package
origin = None
if src_srcinfo is None:
# source package does not exist?
# handle here to avoid crashing on the next line
self.logger.warn("Could not get source info for %s/%s@%s" %
(src_project, src_package, src_rev))
return False
if self.ibs and target_project.startswith('SUSE:SLE'):
if package in self.lookup_sle15:
origin = self.lookup_sle15[package]
origin_same = True
if origin:
origin_same = self._check_same_origin(origin, src_project)
self.logger.info("expected origin is '%s' (%s)", origin,
"unchanged" if origin_same else "changed")
prj = 'openSUSE.org:openSUSE:Factory'
# True or None (open request) are acceptable for SLE.
self.source_in_factory = self._check_factory(
package, src_srcinfo, prj)
if self.source_in_factory is None:
self.pending_factory_submission = True
if self.source_in_factory is not False:
return self.source_in_factory
# got false. could mean package doesn't exist or no match
if self.is_package_in_project(prj, package):
self.logger.info('different sources in {}'.format(
self.rdiff_link(src_project, src_package, src_rev, prj,
package)))
prj = 'openSUSE.org:openSUSE:Leap:15.0'
# TODO Ugly save for SLE-15-SP1.
if False and self.is_package_in_project(prj, package):
if self._check_factory(package, src_srcinfo, prj) is True:
self.logger.info('found source match in {}'.format(prj))
else:
self.logger.info('different sources in {}'.format(
self.rdiff_link(src_project, src_package, src_rev, prj,
package)))
devel_project, devel_package = devel_project_get(
self.apiurl, 'openSUSE.org:openSUSE:Factory', package)
if devel_project is not None:
# specifying devel package is optional
if devel_package is None:
devel_package = package
if self.is_package_in_project(devel_project, devel_package):
if self.factory._check_project(
devel_project, devel_package,
src_srcinfo.verifymd5) == True:
self.logger.info('matching sources in {}/{}'.format(
devel_project, devel_package))
return True
else:
self.logger.info('different sources in {}'.format(
self.rdiff_link(src_project, src_package, src_rev,
devel_project, devel_package)))
else:
self.logger.info('no devel project found for {}/{}'.format(
'openSUSE.org:openSUSE:Factory', package))
#self.logger.info('no matching sources in Factory, Leap:15.0, nor devel project')
self.logger.info(
'no matching sources in Factory, nor devel project')
if origin_same is False:
# Rather than decline, leave review open in-case of change and
# ask release manager for input via override comment.
self.logger.info(
'Comment `(at){} override accept` to force accept.'.format(
self.review_user))
self.needs_release_manager = True
return None
return origin_same
if package in self.lookup_150:
origin = self.lookup_150[package]
# obviously
if src_project in ('openSUSE:Factory', 'openSUSE:Factory:NonFree'):
self.source_in_factory = True
is_fine_if_factory = False
not_in_factory_okish = False
if origin:
origin_same = self._check_same_origin(origin, src_project)
self.logger.info("expected origin is '%s' (%s)", origin,
"unchanged" if origin_same else "changed")
if origin.startswith('Devel;'):
if origin_same == False:
self.logger.debug("not submitted from devel project")
return False
is_fine_if_factory = True
not_in_factory_okish = True
if self.must_approve_version_updates:
self.needs_release_manager = True
# fall through to check history and requests
elif origin.startswith('openSUSE:Factory'):
# A large number of requests are created by hand that leaper
# would have created via update_crawler.py. This applies to
# other origins, but primary looking to let Factory submitters
# know that there is no need to make manual submissions to both.
# Since it has a lookup entry it is not a new package.
self.automatic_submission = False
if self.must_approve_version_updates:
self.needs_release_manager = True
if origin == src_project:
self.source_in_factory = True
# no need to approve submissions from Factory if
# the lookup file points to Factory. Just causes
# spam for many maintainers #1393
self.do_check_maintainer_review = False
is_fine_if_factory = True
# fall through to check history and requests
elif origin == 'FORK':
is_fine_if_factory = True
if not src_project.startswith('SUSE:SLE-'):
not_in_factory_okish = True
self.needs_check_source = True
self.needs_release_manager = True
# fall through to check history and requests
# TODO Ugly save for 15.1 (n-1).
elif False and origin.startswith('openSUSE:Leap:15.0'):
if self.must_approve_maintenance_updates:
self.needs_release_manager = True
# submitted from :Update
if origin_same:
self.logger.debug("submission from 15.0 ok")
return True
# switching to sle package might make sense
if src_project.startswith('SUSE:SLE-15'):
self.needs_release_manager = True
return True
# submitted from elsewhere but is in :Update
else:
good = self.factory._check_project(
'openSUSE:Leap:15.0:Update', target_package,
src_srcinfo.verifymd5)
if good:
self.logger.info("submission found in 15.0")
return good
# check release requests too
good = self.factory._check_requests(
'openSUSE:Leap:15.0:Update', target_package,
src_srcinfo.verifymd5)
if good or good == None:
self.logger.debug("found request")
return good
# let's see where it came from before
if package in self.lookup_150:
oldorigin = self.lookup_150[package]
self.logger.debug("oldorigin {}".format(oldorigin))
# Factory. So it's ok to keep upgrading it to Factory
# TODO: whitelist packages where this is ok and block others?
self.logger.info("Package was from %s in 15.0", oldorigin)
if oldorigin.startswith('openSUSE:Factory'):
# check if an attempt to switch to SLE package is made
for sp in ('SP1:GA', 'SP1:Update'):
good = self.factory._check_project(
'SUSE:SLE-15-{}'.format(sp), target_package,
src_srcinfo.verifymd5)
if good:
self.logger.info(
"request sources come from SLE")
self.needs_release_manager = True
return good
# TODO Ugly save for 15.2 (n-2).
elif False and oldorigin.startswith('openSUSE:Leap:15.0'):
o = self.lookup_150[package]
self.logger.info("Package was from %s in 15.0", o)
# the release manager needs to review attempts to upgrade to Factory
is_fine_if_factory = True
self.needs_release_manager = True
elif origin.startswith('SUSE:SLE-15'):
if self.must_approve_maintenance_updates:
self.needs_release_manager = True
for v in ('15.0', ):
prj = 'openSUSE:Leap:{}:SLE-workarounds'.format(v)
if self.is_package_in_project(prj, target_package):
self.logger.info("found package in %s", prj)
if not self.factory._check_project(
prj, target_package, src_srcinfo.verifymd5):
self.logger.info(
"sources in %s are [NOT identical](%s)", prj,
self.rdiff_link(src_project, src_package,
src_rev, prj, package))
self.needs_release_manager = True
# submitted from :Update
if origin == src_project:
self.logger.debug("submission origin ok")
return True
elif origin.endswith(':GA') \
and src_project == origin[:-2]+'Update':
self.logger.debug("sle update submission")
return True
# check if submitted from higher SP
priolist = [
'SUSE:SLE-15:', 'SUSE:SLE-15-SP1:', 'SUSE:SLE-15-SP2:',
'SUSE:SLE-15-SP3:'
]
for i in range(len(priolist) - 1):
if origin.startswith(priolist[i]):
for prj in priolist[i + 1:]:
if src_project.startswith(prj):
self.logger.info(
"submission from higher service pack %s:* ok",
prj)
return True
in_sle_origin = self._check_factory(target_package,
src_srcinfo, origin)
if in_sle_origin:
self.logger.info(
'parallel submission, also in {}'.format(origin))
return True
self.needs_release_manager = True
# the release manager needs to review attempts to upgrade to Factory
is_fine_if_factory = True
else:
self.logger.error("unhandled origin %s", origin)
return False
else: # no origin
# submission from SLE is ok
if src_project.startswith('SUSE:SLE-15'):
self.do_check_maintainer_review = False
return True
# new package submitted from Factory. Check if it was in
# 42.3 before and skip maintainer review if so.
subprj = src_project[len('openSUSE:Factory'):]
if self.source_in_factory and target_project.startswith('openSUSE:Leap:15.0') \
and self.is_package_in_project('openSUSE:Leap:42.3'+subprj, package):
self.logger.info('package was in 42.3')
self.do_check_maintainer_review = False
return True
is_fine_if_factory = True
self.needs_release_manager = True
if origin is None or not origin.startswith('SUSE:SLE-'):
for p in (':Update', ':GA'):
prj = 'SUSE:SLE-15' + p
if self.is_package_in_project(prj, package):
self.logger.info('Package is in {}'.format(
self.rdiff_link(src_project, src_package, src_rev, prj,
package)))
break
is_in_factory = self.source_in_factory
# we came here because none of the above checks find it good, so
# let's see if the package is in Factory at least
if is_in_factory is None:
is_in_factory = self._check_factory(package, src_srcinfo)
if is_in_factory:
self.source_in_factory = True
self.needs_reviewteam = False
self.needs_legal_review = False
elif is_in_factory is None:
self.pending_factory_submission = True
self.needs_reviewteam = False
self.needs_legal_review = False
else:
if src_project.startswith('SUSE:SLE-15') \
or src_project.startswith('openSUSE:Leap:15.'):
self.needs_reviewteam = False
self.needs_legal_review = False
else:
self.needs_reviewteam = True
self.needs_legal_review = True
self.source_in_factory = False
if is_fine_if_factory:
if self.source_in_factory:
return True
elif self.pending_factory_submission:
return None
elif not_in_factory_okish:
self.needs_reviewteam = True
self.needs_legal_review = True
return True
if self.override_allow:
# Rather than decline, leave review open and ask release
# manager for input via override comment.
self.logger.info(
'Comment `(at){} override accept` to force accept.'.format(
self.review_user))
self.needs_release_manager = True
return None
return False
def crawl(self):
"""Main method"""
succeeded_packages = []
succeeded_packages = self.get_build_succeeded_packages(self.from_prj)
if not len(succeeded_packages) > 0:
logging.info('No build succeeded package in %s'%self.from_prj)
return
# randomize the list
random.shuffle(succeeded_packages)
# get souce packages from target
target_packages = self.get_source_packages(self.to_prj)
deleted_packages = self.get_deleted_packages(self.to_prj)
if self.to_prj.startswith("openSUSE:"):
deleted_packages = deleted_packages + self.get_deleted_packages(OPENSUSE_PREVERSION)
pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, 'openSUSE:Factory')
skip_pkgs_list = self.load_skip_pkgs_list(pseudometa_project, pseudometa_package).splitlines()
ms_packages = [] # collect multi specs packages
for i in range(0, min(int(self.submit_limit), len(succeeded_packages))):
package = succeeded_packages[i]
submit_ok = True
if package in deleted_packages:
logging.info('%s has been dropped from %s, ignore it!'%(package, self.to_prj))
submit_ok = False
if self.is_sle_base_pkgs(package) is True:
logging.info('%s origin from SLE base, skip for now!'%package)
submit_ok = False
# make sure it is new package
new_pkg = self.is_new_package(self.to_prj, package)
if new_pkg is not True:
logging.info('%s is not a new package, do not submit.' % package)
submit_ok = False
multi_specs = self.check_multiple_specfiles(self.factory, package)
if multi_specs is None:
logging.info('%s does not exist in %s'%(package, 'openSUSE:Factory'))
submit_ok = False
if multi_specs:
if multi_specs['linkinfo']:
logging.info('%s in %s is sub-package of %s, skip it!'%(package, 'openSUSE:Factory', multi_specs['linkinfo']))
ms_packages.append(package)
submit_ok = False
for spec in multi_specs['specs']:
if spec not in succeeded_packages:
logging.info('%s is sub-pacakge of %s but build failed, skip it!'%(spec, package))
submit_ok = False
if not submit_ok:
continue
# make sure the package non-exist in target yet ie. expand=False
if package not in target_packages:
# make sure there is no request against same package
request = self.get_request_list(package)
if request:
logging.debug("There is a request to %s / %s already or it has been declined/revoked, skip!"%(package, self.to_prj))
else:
logging.info("%d - Preparing submit %s to %s"%(i, package, self.to_prj))
# get devel project
devel_prj, devel_pkg = devel_project_get(self.apiurl, self.factory, package)
# check devel project does not in the skip list
if devel_prj in self.skip_devel_project_list:
# check the except packages list
match = None
for elem in self.except_pkgs_list:
m = re.search(elem, package)
if m is not None:
match = True
if match is not True:
logging.info('%s/%s is in the skip list, do not submit.' % (devel_prj, package))
continue
else:
pass
# check package does not in the skip list
match = None
for elem in skip_pkgs_list:
m = re.search(elem, package)
if m is not None:
match = True
if match is True:
logging.info('%s is in the skip list, do not submit.' % package)
continue
else:
pass
res = self.create_submitrequest(package)
if res and res is not None:
logging.info('Created request %s for %s' % (res, package))
else:
logging.error('Error occurred when creating submit request')
else:
logging.debug('%s is exist in %s, skip!'%(package, self.to_prj))
# dump multi specs packages
print("Multi-specfile packages:")
if ms_packages:
for pkg in ms_packages:
print pkg
else:
print 'None'
def check_source_submission(self, source_project, source_package,
source_revision, target_project,
target_package):
super(CheckSource,
self).check_source_submission(source_project, source_package,
source_revision, target_project,
target_package)
self.target_project_config(target_project)
if not self.ignore_devel:
self.logger.info(
'checking if target package exists and has devel project')
devel_project, devel_package = devel_project_get(
self.apiurl, target_project, target_package)
if devel_project:
if (source_project != devel_project or source_package != devel_package) and \
not(source_project == target_project and source_package == target_package):
# Not from proper devel project/package and not self-submission.
self.review_messages[
'declined'] = 'Expected submission from devel package %s/%s' % (
devel_project, devel_package)
return False
else:
# Check to see if other packages exist with the same source project
# which indicates that the project has already been used as devel.
if not self.is_devel_project(source_project, target_project):
self.review_messages[
'declined'] = '%s is not a devel project of %s, submit the package to a devel project first' % (
source_project, target_project)
return False
# Checkout and see if renaming package screws up version parsing.
dir = os.path.expanduser('~/co/%s' % self.request.reqid)
if os.path.exists(dir):
self.logger.warn('directory %s already exists' % dir)
shutil.rmtree(dir)
os.makedirs(dir)
os.chdir(dir)
old_info = {'version': None}
try:
CheckSource.checkout_package(self.apiurl,
target_project,
target_package,
pathname=dir,
server_service_files=True,
expand_link=True)
shutil.rmtree(os.path.join(target_package, '.osc'))
os.rename(target_package, '_old')
old_info = self.package_source_parse(target_project,
target_package)
except urllib2.HTTPError:
self.logger.error('failed to checkout %s/%s' %
(target_project, target_package))
CheckSource.checkout_package(self.apiurl,
source_project,
source_package,
revision=source_revision,
pathname=dir,
server_service_files=True,
expand_link=True)
os.rename(source_package, target_package)
shutil.rmtree(os.path.join(target_package, '.osc'))
new_info = self.package_source_parse(source_project, source_package,
source_revision)
if new_info['name'] != target_package:
shutil.rmtree(dir)
self.review_messages[
'declined'] = "A package submitted as %s has to build as 'Name: %s' - found Name '%s'" % (
target_package, target_package, new_info['name'])
return False
# We want to see the same package name in the devel project as in the distro; anything else calls for confusion
if source_package != target_package:
self.review_messages[
'declined'] = "No in-air renames: The package must be called the same in the devel project as in the target project"
return False
# Run check_source.pl script and interpret output.
source_checker = os.path.join(CheckSource.SCRIPT_PATH,
'check_source.pl')
civs = ''
new_version = None
if old_info['version'] and old_info['version'] != new_info['version']:
new_version = new_info['version']
civs += "NEW_VERSION='{}' ".format(new_version)
civs += 'LC_ALL=C perl %s _old %s 2>&1' % (source_checker,
target_package)
p = subprocess.Popen(civs,
shell=True,
stdout=subprocess.PIPE,
close_fds=True)
ret = os.waitpid(p.pid, 0)[1]
checked = p.stdout.readlines()
output = ' '.join(checked).translate(None, '\033')
os.chdir('/tmp')
# ret = 0 : Good
# ret = 1 : Bad
# ret = 2 : Bad but can be non-fatal in some cases
if ret > 1 and target_project.startswith('openSUSE:Leap:') and (
source_project.startswith('SUSE:SLE-15:')
or source_project.startswith('openSUSE:Factory')):
pass
elif ret != 0:
shutil.rmtree(dir)
self.review_messages[
'declined'] = "Output of check script:\n" + output
return False
shutil.rmtree(dir)
self.review_messages['accepted'] = 'Check script succeeded'
if len(checked):
self.review_messages[
'accepted'] += "\n\nOutput of check script (non-fatal):\n" + output
if not self.skip_add_reviews:
if self.review_team is not None:
self.add_review(self.request,
by_group=self.review_team,
msg='Please review sources')
if self.only_changes():
self.logger.debug('only .changes modifications')
staging_group = self.staging_group(target_project)
if staging_group and not self.dryrun:
osc.core.change_review_state(
self.apiurl,
str(self.request.reqid),
'accepted',
by_group=staging_group,
message=
'skipping the staging process since only .changes modifications'
)
elif self.repo_checker is not None:
self.add_review(self.request,
by_user=self.repo_checker,
msg='Please review build success')
return True
def devel_workflow(self, only_devel):
self.remote_config_set_age_minimum()
devel_project = self.randomString('devel')
package = self.randomString('package')
request = self.wf.create_submit_request(devel_project, package)
attribute_value_save(self.wf.apiurl, devel_project, 'ApprovedRequestSource', '', 'OBS')
if not only_devel:
self.assertReviewBot(request.reqid, self.bot_user, 'new', 'new')
comment = [
'<!-- OriginManager state=seen result=None -->',
'Source not found in allowed origins:',
f'- {self.product_project}',
f'Decision may be overridden via `@{self.bot_user} override`.',
]
self.assertComment(request.reqid, comment)
CommentAPI(self.wf.api.apiurl).add_comment(
request_id=request.reqid, comment=f'@{self.bot_user} change_devel')
comment = 'change_devel command by {}'.format('Admin')
else:
comment = 'only devel origin allowed'
self.assertReviewBot(request.reqid, self.bot_user, 'new', 'accepted')
self.assertAnnotation(request.reqid, {
'comment': comment,
'origin': devel_project,
})
request.change_state('accepted')
memoize_session_reset()
self.osc_user(self.bot_user)
request_future = origin_update(self.wf.apiurl, self.wf.project, package)
self.assertNotEqual(request_future, False)
if request_future:
request_id_change_devel = request_future.print_and_create()
# Ensure a second request is not triggered.
request_future = origin_update(self.wf.apiurl, self.wf.project, package)
self.assertEqual(request_future, False)
self.osc_user_pop()
memoize_session_reset()
origin_info = origin_find(self.wf.apiurl, self.wf.project, package)
self.assertEqual(origin_info, None)
self.assertReviewBot(request_id_change_devel, self.bot_user, 'new', 'accepted')
self.assertAnnotation(request_id_change_devel, {
'origin': devel_project,
})
# Origin should change before request is accepted since it is properly
# annotated and without fallback review.
memoize_session_reset()
origin_info = origin_find(self.wf.apiurl, self.wf.project, package)
self.assertEqual(str(origin_info), devel_project)
self.wf.projects[devel_project].packages[0].create_commit()
self.osc_user(self.bot_user)
request_future = origin_update(self.wf.apiurl, self.wf.project, package)
self.assertNotEqual(request_future, False)
if request_future:
request_id_update = request_future.print_and_create()
request_future = origin_update(self.wf.apiurl, self.wf.project, package)
self.assertEqual(request_future, False)
self.osc_user_pop()
self.assertReviewBot(request_id_update, self.bot_user, 'new', 'accepted')
self.assertAnnotation(request_id_update, {
'origin': devel_project,
})
memoize_session_reset()
devel_project_actual, _ = devel_project_get(self.wf.apiurl, self.wf.project, package)
self.assertEqual(devel_project_actual, None)
request = get_request(self.wf.apiurl, request_id_change_devel)
request_state_change(self.wf.apiurl, request_id_change_devel, 'accepted')
memoize_session_reset()
devel_project_actual, devel_package_actual = devel_project_get(
self.wf.apiurl, self.wf.project, package)
self.assertEqual(devel_project_actual, devel_project)
self.assertEqual(devel_package_actual, package)
request = get_request(self.wf.apiurl, request_id_update)
request_state_change(self.wf.apiurl, request_id_update, 'accepted')
devel_project_new = self.randomString('develnew')
self.wf.create_package(devel_project_new, package)
attribute_value_save(self.wf.apiurl, devel_project_new, 'ApprovedRequestSource', '', 'OBS')
copy_package(self.wf.apiurl, devel_project, package,
self.wf.apiurl, devel_project_new, package)
request_future = request_create_change_devel(
self.wf.apiurl, devel_project_new, package, self.wf.project)
self.assertNotEqual(request_future, False)
if request_future:
request_id_change_devel_new = request_future.print_and_create()
self.assertReviewBot(request_id_change_devel_new, self.bot_user, 'new', 'accepted')
self.assertAnnotation(request_id_change_devel_new, {
'origin': devel_project_new,
'origin_old': devel_project,
})
self.accept_fallback_review(request_id_change_devel_new)
request_state_change(self.wf.apiurl, request_id_change_devel_new, 'accepted')
memoize_session_reset()
origin_info = origin_find(self.wf.apiurl, self.wf.project, package)
self.assertEqual(str(origin_info), devel_project_new)
def check_source_submission(self, source_project, source_package,
source_revision, target_project,
target_package):
super(CheckSource,
self).check_source_submission(source_project, source_package,
source_revision, target_project,
target_package)
self.target_project_config(target_project)
if self.single_action_require and len(self.request.actions) != 1:
self.review_messages[
'declined'] = 'Only one action per request allowed'
return False
kind = package_kind(self.apiurl, target_project, target_package)
if kind == 'meta':
self.review_messages[
'accepted'] = 'Skipping all checks for meta packages'
return True
elif (kind is not None and kind != 'source'):
self.review_messages[
'declined'] = 'May not modify a non-source package of type {}'.format(
kind)
return False
inair_renamed = target_package != source_package
if not self.ignore_devel:
self.logger.info(
'checking if target package exists and has devel project')
devel_project, devel_package = devel_project_get(
self.apiurl, target_project, target_package)
if devel_project:
if (source_project != devel_project or source_package != devel_package) and \
not(source_project == target_project and source_package == target_package):
# Not from proper devel project/package and not self-submission.
self.review_messages[
'declined'] = 'Expected submission from devel package %s/%s' % (
devel_project, devel_package)
return False
else:
# Check to see if other packages exist with the same source project
# which indicates that the project has already been used as devel.
if not self.is_devel_project(source_project, target_project):
self.review_messages['declined'] = (
'%s is not a devel project of %s, submit the package to a devel project first. '
'See https://en.opensuse.org/openSUSE:How_to_contribute_to_Factory#How_to_request_a_new_devel_project for details.'
) % (source_project, target_project)
return False
else:
if source_project.endswith(':Update'):
# Allow for submission like:
# - source: openSUSE:Leap:15.0:Update/google-compute-engine.8258
# - target: openSUSE:Leap:15.1/google-compute-engine
# Note: home:jberry:Update would also be allowed via this condition,
# but that should be handled by leaper and human review.
# Ignore a dot in package name (ex. tpm2.0-abrmd) and instead
# only look for ending in dot number.
match = re.match(r'(.*)\.\d+$', source_package)
if match:
inair_renamed = target_package != match.group(1)
if not self.in_air_rename_allow and inair_renamed:
self.review_messages[
'declined'] = 'Source and target package names must match'
return False
# Checkout and see if renaming package screws up version parsing.
dir = os.path.expanduser('~/co/%s' % self.request.reqid)
if os.path.exists(dir):
self.logger.warning('directory %s already exists' % dir)
shutil.rmtree(dir)
os.makedirs(dir)
os.chdir(dir)
old_info = {'version': None}
try:
CheckSource.checkout_package(self.apiurl,
target_project,
target_package,
pathname=dir,
server_service_files=True,
expand_link=True)
shutil.rmtree(os.path.join(target_package, '.osc'))
os.rename(target_package, '_old')
old_info = self.package_source_parse(target_project,
target_package)
except HTTPError as e:
if e.code == 404:
self.logger.info('target package does not exist %s/%s' %
(target_project, target_package))
else:
raise e
CheckSource.checkout_package(self.apiurl,
source_project,
source_package,
revision=source_revision,
pathname=dir,
server_service_files=True,
expand_link=True)
os.rename(source_package, target_package)
shutil.rmtree(os.path.join(target_package, '.osc'))
new_info = self.package_source_parse(source_project, source_package,
source_revision, target_package)
filename = new_info.get('filename', '')
if not (filename.endswith('.kiwi') or filename
== 'Dockerfile') and new_info['name'] != target_package:
shutil.rmtree(dir)
self.review_messages[
'declined'] = "A package submitted as %s has to build as 'Name: %s' - found Name '%s'" % (
target_package, target_package, new_info['name'])
return False
# Run check_source.pl script and interpret output.
source_checker = os.path.join(CheckSource.SCRIPT_PATH,
'check_source.pl')
civs = ''
new_version = None
if old_info['version'] and old_info['version'] != new_info['version']:
new_version = new_info['version']
civs += "NEW_VERSION='{}' ".format(new_version)
civs += 'LC_ALL=C perl %s _old %s 2>&1' % (source_checker,
target_package)
p = subprocess.Popen(civs,
shell=True,
stdout=subprocess.PIPE,
close_fds=True)
ret = os.waitpid(p.pid, 0)[1]
checked = decode_list(p.stdout.readlines())
output = ' '.join(checked).replace('\033', '')
os.chdir('/tmp')
# ret = 0 : Good
# ret = 1 : Bad
# ret = 2 : Bad but can be non-fatal in some cases
if ret > 1 and target_project.startswith('openSUSE:Leap:') and (
source_project.startswith('SUSE:SLE-15:')
or source_project.startswith('openSUSE:Factory')):
pass
elif ret != 0:
shutil.rmtree(dir)
self.review_messages[
'declined'] = "Output of check script:\n" + output
return False
shutil.rmtree(dir)
self.review_messages['accepted'] = 'Check script succeeded'
if len(checked):
self.review_messages[
'accepted'] += "\n\nOutput of check script (non-fatal):\n" + output
if not self.skip_add_reviews:
if self.add_review_team and self.review_team is not None:
self.add_review(self.request,
by_group=self.review_team,
msg='Please review sources')
if self.only_changes():
self.logger.debug('only .changes modifications')
if self.staging_group and self.review_user in group_members(
self.apiurl, self.staging_group):
if not self.dryrun:
osc.core.change_review_state(
self.apiurl,
str(self.request.reqid),
'accepted',
by_group=self.staging_group,
message=
'skipping the staging process since only .changes modifications'
)
else:
self.logger.debug(
'unable to skip staging review since not a member of staging group'
)
elif self.repo_checker is not None:
self.add_review(self.request,
by_user=self.repo_checker,
msg='Please review build success')
if self.bad_rpmlint_entries:
if self.has_whitelist_warnings(source_project, source_package,
target_project, target_package):
# if there are any add a review for the security team
# maybe add the found warnings to the message for the review
self.add_review(self.request,
by_group=self.security_review_team,
msg=CheckSource.AUDIT_BUG_MESSAGE)
if self.suppresses_whitelist_warnings(source_project,
source_package):
self.add_review(self.request,
by_group=self.security_review_team,
msg=CheckSource.AUDIT_BUG_MESSAGE)
return True
def check_source_submission(self, source_project, source_package, source_revision, target_project, target_package):
super(CheckSource, self).check_source_submission(source_project, source_package, source_revision, target_project, target_package)
self.target_project_config(target_project)
if self.single_action_require and len(self.request.actions) != 1:
self.review_messages['declined'] = 'Only one action per request allowed'
return False
if target_package.startswith('00') or target_package.startswith('_'):
self.review_messages['accepted'] = 'Skipping all checks for product related packages'
return True
inair_renamed = target_package != source_package
if not self.ignore_devel:
self.logger.info('checking if target package exists and has devel project')
devel_project, devel_package = devel_project_get(self.apiurl, target_project, target_package)
if devel_project:
if (source_project != devel_project or source_package != devel_package) and \
not(source_project == target_project and source_package == target_package):
# Not from proper devel project/package and not self-submission.
self.review_messages['declined'] = 'Expected submission from devel package %s/%s' % (devel_project, devel_package)
return False
else:
# Check to see if other packages exist with the same source project
# which indicates that the project has already been used as devel.
if not self.is_devel_project(source_project, target_project):
self.review_messages['declined'] = (
'%s is not a devel project of %s, submit the package to a devel project first. '
'See https://en.opensuse.org/openSUSE:How_to_contribute_to_Factory#How_to_request_a_new_devel_project for details.'
) % (source_project, target_project)
return False
else:
if source_project.endswith(':Update'):
# Allow for submission like:
# - source: openSUSE:Leap:15.0:Update/google-compute-engine.8258
# - target: openSUSE:Leap:15.1/google-compute-engine
# Note: home:jberry:Update would also be allowed via this condition,
# but that should be handled by leaper and human review.
# Ignore a dot in package name (ex. tpm2.0-abrmd) and instead
# only look for ending in dot number.
match = re.match(r'(.*)\.\d+$', source_package)
if match:
inair_renamed = target_package != match.group(1)
if not self.in_air_rename_allow and inair_renamed:
self.review_messages['declined'] = 'Source and target package names must match'
return False
# Checkout and see if renaming package screws up version parsing.
dir = os.path.expanduser('~/co/%s' % self.request.reqid)
if os.path.exists(dir):
self.logger.warn('directory %s already exists' % dir)
shutil.rmtree(dir)
os.makedirs(dir)
os.chdir(dir)
old_info = {'version': None}
try:
CheckSource.checkout_package(self.apiurl, target_project, target_package, pathname=dir,
server_service_files=True, expand_link=True)
shutil.rmtree(os.path.join(target_package, '.osc'))
os.rename(target_package, '_old')
old_info = self.package_source_parse(target_project, target_package)
except HTTPError:
self.logger.error('failed to checkout %s/%s' % (target_project, target_package))
CheckSource.checkout_package(self.apiurl, source_project, source_package, revision=source_revision,
pathname=dir, server_service_files=True, expand_link=True)
os.rename(source_package, target_package)
shutil.rmtree(os.path.join(target_package, '.osc'))
new_info = self.package_source_parse(source_project, source_package, source_revision)
if new_info['name'] != target_package:
shutil.rmtree(dir)
self.review_messages['declined'] = "A package submitted as %s has to build as 'Name: %s' - found Name '%s'" % (target_package, target_package, new_info['name'])
return False
# Run check_source.pl script and interpret output.
source_checker = os.path.join(CheckSource.SCRIPT_PATH, 'check_source.pl')
civs = ''
new_version = None
if old_info['version'] and old_info['version'] != new_info['version']:
new_version = new_info['version']
civs += "NEW_VERSION='{}' ".format(new_version)
civs += 'LC_ALL=C perl %s _old %s 2>&1' % (source_checker, target_package)
p = subprocess.Popen(civs, shell=True, stdout=subprocess.PIPE, close_fds=True)
ret = os.waitpid(p.pid, 0)[1]
checked = p.stdout.readlines()
output = ' '.join(checked).translate(None, '\033')
os.chdir('/tmp')
# ret = 0 : Good
# ret = 1 : Bad
# ret = 2 : Bad but can be non-fatal in some cases
if ret > 1 and target_project.startswith('openSUSE:Leap:') and (source_project.startswith('SUSE:SLE-15:') or source_project.startswith('openSUSE:Factory')):
pass
elif ret != 0:
shutil.rmtree(dir)
self.review_messages['declined'] = "Output of check script:\n" + output
return False
shutil.rmtree(dir)
self.review_messages['accepted'] = 'Check script succeeded'
if len(checked):
self.review_messages['accepted'] += "\n\nOutput of check script (non-fatal):\n" + output
if not self.skip_add_reviews:
if self.add_review_team and self.review_team is not None:
self.add_review(self.request, by_group=self.review_team, msg='Please review sources')
if self.only_changes():
self.logger.debug('only .changes modifications')
if self.staging_group and self.review_user in group_members(self.apiurl, self.staging_group):
if not self.dryrun:
osc.core.change_review_state(self.apiurl, str(self.request.reqid), 'accepted',
by_group=self.staging_group,
message='skipping the staging process since only .changes modifications')
else:
self.logger.debug('unable to skip staging review since not a member of staging group')
elif self.repo_checker is not None:
self.add_review(self.request, by_user=self.repo_checker, msg='Please review build success')
return True
