def package_comments(self, project): self.logger.info('{} package comments'.format(len( self.package_results))) for package, sections in self.package_results.items(): message = 'The version of this package in `{}` has installation issues and may not be installable:'.format( project) # Sort sections by text to group binaries together. sections = sorted(sections, key=lambda s: s.text) message += '\n\n<pre>\n{}\n</pre>'.format('\n'.join( [section.text for section in sections]).strip()) # Generate a hash based on the binaries involved and the number of # sections. This eliminates version or release changes from causing # an update to the comment while still updating on relevant changes. binaries = set() for section in sections: binaries.update(section.binaries) info = ';'.join(['::'.join(sorted(binaries)), str(len(sections))]) reference = hashlib.sha1(info).hexdigest()[:7] # Post comment on devel package in order to notifiy maintainers. devel_project, devel_package = devel_project_get( self.apiurl, project, package) self.comment_write(state='seen', result=reference, project=devel_project, package=devel_package, message=message)
def origin_devel_project(apiurl, project, package): if devel_project_simulate.lock: devel_project, devel_package = devel_project_simulate.lock.get( apiurl, project, package) if devel_project: return devel_project, devel_package for devel_project, devel_package in origin_devel_project_requests( apiurl, project, package): return devel_project, devel_package return devel_project_get(apiurl, project, package)
def check_action_add_role(self, request, action): # Decline add_role request (assumed the bot acting on requests to Factory or similar). message = 'Roles to packages are granted in the devel project, not in %s.' % action.tgt_project if action.tgt_package is not None: project, package = devel_project_get(self.apiurl, action.tgt_project, action.tgt_package) message += ' Send this request to {}/{}.'.format(project, package) self.review_messages['declined'] = message return False
def origin_maintainer_review_ensure(self, origin_info, package, message, request=None): if not request: request = self.request origin = origin_workaround_strip(origin_info.project) # Check if the origin package is actually developed elsewhere and add # the maintainer review for the development location. devel_project, _ = devel_project_get(self.apiurl, origin, package) if devel_project: origin = devel_project users = package_role_expand(self.apiurl, origin, package, 'maintainer') if request.creator not in users: self.add_review(request, by_project=origin, by_package=package, msg=message)
def config_origin_generator(origins, apiurl=None, project=None, package=None, skip_workarounds=False): for origin_item in origins: for origin, values in origin_item.items(): is_workaround = origin_workaround_check(origin) if skip_workarounds and is_workaround: break if (origin == '<devel>' or origin == '<devel>~') and apiurl and project and package: devel_project, devel_package = devel_project_get(apiurl, project, package) if not devel_project: break origin = devel_project if is_workaround: origin = origin_workaround_ensure(origin) yield origin, values break # Only support single value inside list item.
def check_source_submission_inner(self, src_project, src_package, src_rev, target_project, target_package): super(Leaper, self).check_source_submission(src_project, src_package, src_rev, target_project, target_package) self.automatic_submission = False if src_project == target_project and src_package == target_package: self.logger.info('self submission detected') self.needs_release_manager = True return True src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev) package = target_package origin = self.lookup.get(target_project, package) origin_same = True if origin: origin_same = self._check_same_origin(origin, src_project) if src_srcinfo is None: # source package does not exist? # handle here to avoid crashing on the next line self.logger.warn("Could not get source info for %s/%s@%s" % (src_project, src_package, src_rev)) return False if self.ibs and target_project.startswith('SUSE:SLE'): review_result = None prj = 'openSUSE.org:openSUSE:Factory' # True or None (open request) are acceptable for SLE. in_factory = self._check_factory(package, src_srcinfo, prj) if in_factory: review_result = True self.source_in_factory = True elif in_factory is None: self.pending_factory_submission = True else: if not self.is_package_in_project(prj, package): self.logger.info('the package is not in Factory, nor submitted there') else: self.logger.info('different sources in {}'.format(self.rdiff_link(src_project, src_package, src_rev, prj, package))) if review_result == None and not self.pending_factory_submission: other_projects_to_check = [] m = re.match(r'SUSE:SLE-(\d+)(?:-SP(\d+)):', target_project) if m: sle_version = int(m.group(1)) sp_version = int(m.group(2)) versions_to_check = [] # yeah, too much harcoding here if sle_version == 12: versions_to_check = [ '42.3' ] elif sle_version == 15: versions_to_check = [ '15.%d'%i for i in range(sp_version+1) ] else: self.logger.error("can't handle %d.%d", sle_version, sp_version) for version in versions_to_check: leap = 'openSUSE.org:openSUSE:Leap:%s'%(version) other_projects_to_check += [ leap, leap + ':Update', leap + ':NonFree', leap + ':NonFree:Update' ] for prj in other_projects_to_check: if self.is_package_in_project(prj, package): self.logger.debug('checking {}'.format(prj)) if self._check_factory(package, src_srcinfo, prj) is True: self.logger.info('found source match in {}'.format(prj)) devel_project, devel_package = devel_project_get(self.apiurl, 'openSUSE.org:openSUSE:Factory', package) if devel_project is not None: # specifying devel package is optional if devel_package is None: devel_package = package if self.is_package_in_project(devel_project, devel_package): if self._check_matching_srcmd5(devel_project, devel_package, src_srcinfo.verifymd5) == True: self.logger.info('matching sources in {}/{}'.format(devel_project, devel_package)) return True else: self.logger.info('different sources in devel project {}'.format(self.rdiff_link(src_project, src_package, src_rev, devel_project, devel_package))) else: self.logger.info('no devel project found for {}/{}'.format('openSUSE.org:openSUSE:Factory', package)) self.logger.info('no matching sources found anywhere. Needs a human to decide whether that is ok. Please provide some justification to help that person.') else: leap = 'openSUSE.org:openSUSE:Leap:15.1' if not self.is_package_in_project(target_project, package) \ and self.is_package_in_project(leap, package) \ and self._check_factory(package, src_srcinfo, leap) is False: self.logger.info('different sources in {}'.format(self.rdiff_link(src_project, src_package, src_rev, leap, package))) if not review_result and origin is not None: review_result = origin_same if origin_same or origin == 'openSUSE.org:openSUSE:Factory' and self.pending_factory_submission: self.logger.info("ok, origin %s unchanged", origin) else: # only log origin state if it's taken into consideration for the review result self.logger.info("Submitted from a different origin than expected ('%s')", origin) self.needs_release_manager = True # no result so far and also no factory submission to wait # for. So just pass to avoid requring too much overrides if not self.pending_factory_submission: review_result = True if not review_result and self.override_allow: # Rather than decline, leave review open in-case of change and # ask release manager for input via override comment. self.logger.info('Comment `(at){} override accept` to force accept.'.format(self.review_user)) self.needs_release_manager = True review_result = None return review_result if target_project.endswith(':Update'): self.logger.info("expected origin is '%s' (%s)", origin, "unchanged" if origin_same else "changed") # Only when not from current product should request require maintainer review. self.do_check_maintainer_review = False if origin_same: return True good = self._check_matching_srcmd5(origin, target_package, src_srcinfo.verifymd5) if good: self.logger.info('submission source found in origin ({})'.format(origin)) return good good = self.factory._check_requests(origin, target_package, src_srcinfo.verifymd5) if good or good == None: self.logger.info('found pending submission against origin ({})'.format(origin)) return good # TODO #1662: Uncomment once maintbot has been superseded and leaper # is no longer run in comment-only mode. #self.do_check_maintainer_review = True return None elif self.action.type == 'maintenance_incident': self.logger.debug('unhandled incident pattern (targetting non :Update project)') return True # obviously if src_project in ('openSUSE:Factory', 'openSUSE:Factory:NonFree'): self.source_in_factory = True is_fine_if_factory = False not_in_factory_okish = False if origin: self.logger.info("expected origin is '%s' (%s)", origin, "unchanged" if origin_same else "changed") if origin.startswith('Devel;'): if origin_same == False: self.logger.debug("not submitted from devel project") return False is_fine_if_factory = True not_in_factory_okish = True if self.must_approve_version_updates: self.needs_release_manager = True # fall through to check history and requests elif origin.startswith('openSUSE:Factory'): # A large number of requests are created by hand that leaper # would have created via update_crawler.py. This applies to # other origins, but primary looking to let Factory submitters # know that there is no need to make manual submissions to both. # Since it has a lookup entry it is not a new package. self.automatic_submission = False if self.must_approve_version_updates: self.needs_release_manager = True if origin == src_project: self.source_in_factory = True # no need to approve submissions from Factory if # the lookup file points to Factory. Just causes # spam for many maintainers #1393 self.do_check_maintainer_review = False is_fine_if_factory = True # fall through to check history and requests elif origin == 'FORK': is_fine_if_factory = True if not src_project.startswith('SUSE:SLE-'): not_in_factory_okish = True self.needs_check_source = True self.needs_release_manager = True # fall through to check history and requests # TODO Ugly save for 15.1 (n-1). elif origin.startswith('openSUSE:Leap:15.0'): if self.must_approve_maintenance_updates: self.needs_release_manager = True if src_project.startswith('openSUSE:Leap'): self.do_check_maintainer_review = False # submitted from :Update if origin_same: self.logger.debug("submission from 15.0 ok") return True # switching to sle package might make sense if src_project.startswith('SUSE:SLE-15'): self.needs_release_manager = True self.do_check_maintainer_review = False return True # submitted from elsewhere but is in :Update else: good = self._check_matching_srcmd5('openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5) if good: self.logger.info("submission found in 15.0") return good # check release requests too good = self.factory._check_requests('openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5) if good or good == None: self.logger.debug("found request") return good # let's see where it came from before oldorigin = self.lookup.get('openSUSE:Leap:15.0', target_package) if oldorigin: self.logger.debug("oldorigin {}".format(oldorigin)) # Factory. So it's ok to keep upgrading it to Factory # TODO: whitelist packages where this is ok and block others? self.logger.info("Package was from %s in 15.0", oldorigin) if oldorigin.startswith('openSUSE:Factory'): # check if an attempt to switch to SLE package is made for sp in ('SP1:GA', 'SP1:Update'): good = self._check_matching_srcmd5('SUSE:SLE-15-{}'.format(sp), target_package, src_srcinfo.verifymd5) if good: self.logger.info("request sources come from SLE") self.needs_release_manager = True return good # TODO Ugly save for 15.2 (n-2). elif False and oldorigin.startswith('openSUSE:Leap:15.0'): self.logger.info("Package was from %s in 15.0", oldorigin) # the release manager needs to review attempts to upgrade to Factory is_fine_if_factory = True self.needs_release_manager = True elif origin.startswith('SUSE:SLE-15'): if self.must_approve_maintenance_updates: self.needs_release_manager = True if src_project.startswith('SUSE:SLE-15'): self.do_check_maintainer_review = False for v in ('15.0', '15.1'): prj = 'openSUSE:Leap:{}:SLE-workarounds'.format(v) if self.is_package_in_project( prj, target_package): self.logger.info("found package in %s", prj) if not self._check_matching_srcmd5(prj, target_package, src_srcinfo.verifymd5): self.logger.info("sources in %s are NOT identical", self.rdiff_link(src_project, src_package, src_rev, prj, package)) self.needs_release_manager = True # submitted from :Update if origin == src_project: self.logger.debug("submission origin ok") return True elif origin.endswith(':GA') \ and src_project == origin[:-2]+'Update': self.logger.debug("sle update submission") return True # check if submitted from higher SP priolist = ['SUSE:SLE-15:', 'SUSE:SLE-15-SP1:', 'SUSE:SLE-15-SP2:', 'SUSE:SLE-15-SP3:'] for i in range(len(priolist)-1): if origin.startswith(priolist[i]): for prj in priolist[i+1:]: if src_project.startswith(prj): self.logger.info("submission from higher service pack %s:* ok", prj) return True in_sle_origin = self._check_factory(target_package, src_srcinfo, origin) if in_sle_origin: self.logger.info('parallel submission, also in {}'.format(origin)) return True self.needs_release_manager = True # the release manager needs to review attempts to upgrade to Factory is_fine_if_factory = True else: self.logger.error("unhandled origin %s", origin) return False else: # no origin # submission from SLE is ok if src_project.startswith('SUSE:SLE-15'): self.do_check_maintainer_review = False return True # new package submitted from Factory. Check if it was in # 42.3 before and skip maintainer review if so. subprj = src_project[len('openSUSE:Factory'):] # disabled for reference. Needed again for 16.0 probably if False and self.source_in_factory and target_project.startswith('openSUSE:Leap:15.0') \ and self.is_package_in_project('openSUSE:Leap:42.3'+subprj, package): self.logger.info('package was in 42.3') self.do_check_maintainer_review = False return True is_fine_if_factory = True self.needs_release_manager = True if origin is None or not origin.startswith('SUSE:SLE-'): for p in (':Update', ':GA'): prj = 'SUSE:SLE-15' + p if self.is_package_in_project(prj, package): self.logger.info('Package is in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) break is_in_factory = self.source_in_factory # we came here because none of the above checks find it good, so # let's see if the package is in Factory at least if is_in_factory is None: is_in_factory = self._check_factory(package, src_srcinfo) if is_in_factory: self.source_in_factory = True self.needs_reviewteam = False self.needs_legal_review = False elif is_in_factory is None: self.pending_factory_submission = True self.needs_reviewteam = False self.needs_legal_review = False else: if src_project.startswith('SUSE:SLE-15') \ or src_project.startswith('openSUSE:Leap:15.'): self.needs_reviewteam = False self.needs_legal_review = False else: self.needs_reviewteam = True self.needs_legal_review = True self.source_in_factory = False if is_fine_if_factory: if self.source_in_factory: return True elif self.pending_factory_submission: return None elif not_in_factory_okish: self.needs_reviewteam = True self.needs_legal_review = True return True if self.override_allow: # Rather than decline, leave review open and ask release # manager for input via override comment. self.logger.info('Comment `(at){} override accept` to force accept.'.format(self.review_user)) self.needs_release_manager = True return None return False
def update_targets(self, targets, sources): # special case maintenance project. Only consider main # package names. The code later follows the link in the # source project then. if self.is_maintenance_project(self.from_prj): mainpacks = set() for package, sourceinfo in sources.items(): if package.startswith('patchinfo.'): continue files = set([node.text for node in sourceinfo.findall('filename')]) if '{}.spec'.format(package) in files: mainpacks.add(package) sources = { package: sourceinfo for package, sourceinfo in sources.iteritems() if package in mainpacks } for package, sourceinfo in sources.items(): origin = self.lookup.get(package, '') if origin.startswith('Devel;'): (dummy, origin, dummy) = origin.split(';') if self.filter_lookup and not origin in self.filter_lookup: if not origin.startswith('subpackage of'): self.skipped.setdefault(origin, set()).add(package) continue if not package in targets: if not self.submit_new: logging.info('Package %s not found in targets' % (package)) continue if self.is_source_innerlink(self.from_prj, package): logging.debug('Package %s is sub package' % (package)) continue else: targetinfo = targets[package] # XXX: make more generic :-) devel_prj = devel_project_get(self.apiurl, FACTORY, package) if devel_prj == 'devel:languages:haskell': logging.info('skipping haskell package %s' % package) continue # Compare verifymd5 md5_from = sourceinfo.get('verifymd5') md5_to = targetinfo.get('verifymd5') if md5_from == md5_to: #logging.info('Package %s not marked for update' % package) continue if self.is_source_innerlink(self.to_prj, package): logging.debug('Package %s is sub package' % (package)) continue # this makes only sense if we look at the expanded view # and want to submit from proper project # originproject = default_origin # if not sourceinfo.find('originproject') is None: # originproject = sourceinfo.find('originproject').text # logging.warn('changed originproject for {} to {}'.format(package, originproject)) src_project, src_package, src_rev = self.follow_link(self.from_prj, package, sourceinfo.get('srcmd5'), sourceinfo.get('verifymd5')) res = self.submitrequest(src_project, src_package, src_rev, package, origin) if res: logging.info('Created request %s for %s' % (res, package)) elif res != 0: logging.error('Error creating the request for %s' % package)
def crawl(self): """Main method""" succeeded_packages = [] succeeded_packages = self.get_build_succeeded_packages(self.from_prj) if not len(succeeded_packages) > 0: logging.info('No build succeeded package in %s' % self.from_prj) return # randomize the list random.shuffle(succeeded_packages) # get souce packages from target target_packages = self.get_source_packages(self.to_prj) deleted_packages = self.get_deleted_packages(self.to_prj) if self.to_prj.startswith("openSUSE:"): for prd in OPENSUSE_RELEASED_VERSION: deleted_packages = deleted_packages + self.get_deleted_packages(prd) pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, 'openSUSE:Factory') skip_pkgs_list = self.load_skip_pkgs_list(pseudometa_project, pseudometa_package).splitlines() ms_packages = [] # collect multi specs packages for i in range(0, min(int(self.submit_limit), len(succeeded_packages))): package = succeeded_packages[i] submit_ok = True if package in deleted_packages: logging.info('%s has been dropped from %s, ignore it!' % (package, self.to_prj)) submit_ok = False if self.is_sle_base_pkgs(package) is True: logging.info('%s origin from SLE base, skip for now!' % package) submit_ok = False # make sure it is new package new_pkg = self.is_new_package(self.to_prj, package) if new_pkg is not True: logging.info('%s is not a new package, do not submit.' % package) submit_ok = False multi_specs = self.check_multiple_specfiles(self.factory, package) if multi_specs is None: logging.info('%s does not exist in %s' % (package, 'openSUSE:Factory')) submit_ok = False if multi_specs: if multi_specs['linkinfo']: logging.info('%s in %s is sub-package of %s, skip it!' % (package, 'openSUSE:Factory', multi_specs['linkinfo'])) ms_packages.append(package) submit_ok = False for spec in multi_specs['specs']: if spec not in succeeded_packages: logging.info('%s is sub-pacakge of %s but build failed, skip it!' % (spec, package)) submit_ok = False if not submit_ok: continue # make sure the package non-exist in target yet ie. expand=False if package not in target_packages: # make sure there is no request against same package request = self.get_request_list(package) if request: logging.debug("There is a request to %s / %s already or it has been declined/revoked, skip!" % (package, self.to_prj)) else: logging.info("%d - Preparing submit %s to %s" % (i, package, self.to_prj)) # get devel project devel_prj, devel_pkg = devel_project_get(self.apiurl, self.factory, package) # check devel project does not in the skip list if devel_prj in self.skip_devel_project_list: # check the except packages list match = None for elem in self.except_pkgs_list: m = re.search(elem, package) if m is not None: match = True if match is not True: logging.info('%s/%s is in the skip list, do not submit.' % (devel_prj, package)) continue else: pass # check package does not in the skip list match = None for elem in skip_pkgs_list: m = re.search(str(elem), package) if m is not None: match = True if match is True: logging.info('%s is in the skip list, do not submit.' % package) continue else: pass res = self.create_submitrequest(package) if res and res is not None: logging.info('Created request %s for %s' % (res, package)) else: logging.error('Error occurred when creating submit request') else: logging.debug('%s is exist in %s, skip!' % (package, self.to_prj)) time.sleep(5) # dump multi specs packages print("Multi-specfile packages:") if ms_packages: for pkg in ms_packages: print(pkg) else: print('None')
def update_targets(self, targets, sources): # special case maintenance project. Only consider main # package names. The code later follows the link in the # source project then. if self.is_maintenance_project(self.from_prj): mainpacks = set() for package, sourceinfo in sources.items(): if package.startswith('patchinfo.'): continue files = set( [node.text for node in sourceinfo.findall('filename')]) if '{}.spec'.format(package) in files: mainpacks.add(package) sources = { package: sourceinfo for package, sourceinfo in sources.iteritems() if package in mainpacks } for package, sourceinfo in sources.items(): origin = self.lookup.get(package, '') if origin.startswith('Devel;'): (dummy, origin, dummy) = origin.split(';') if self.filter_lookup and not origin in self.filter_lookup: if not origin.startswith('subpackage of'): self.skipped.setdefault(origin, set()).add(package) continue if not package in targets: if not self.submit_new: logging.info('Package %s not found in targets' % (package)) continue if self.is_source_innerlink(self.from_prj, package): logging.debug('Package %s is sub package' % (package)) continue else: targetinfo = targets[package] # XXX: make more generic :-) devel_prj = devel_project_get(self.apiurl, FACTORY, package) if devel_prj == 'devel:languages:haskell': logging.info('skipping haskell package %s' % package) continue # Compare verifymd5 md5_from = sourceinfo.get('verifymd5') md5_to = targetinfo.get('verifymd5') if md5_from == md5_to: #logging.info('Package %s not marked for update' % package) continue if self.is_source_innerlink(self.to_prj, package): logging.debug('Package %s is sub package' % (package)) continue # this makes only sense if we look at the expanded view # and want to submit from proper project # originproject = default_origin # if not sourceinfo.find('originproject') is None: # originproject = sourceinfo.find('originproject').text # logging.warn('changed originproject for {} to {}'.format(package, originproject)) src_project, src_package, src_rev = self.follow_link( self.from_prj, package, sourceinfo.get('srcmd5'), sourceinfo.get('verifymd5')) res = self.submitrequest(src_project, src_package, src_rev, package, origin) if res: logging.info('Created request %s for %s' % (res, package)) elif res != 0: logging.error('Error creating the request for %s' % package)
def check_source_submission_inner(self, src_project, src_package, src_rev, target_project, target_package): super(Leaper, self).check_source_submission(src_project, src_package, src_rev, target_project, target_package) self.automatic_submission = False if src_project == target_project and src_package == target_package: self.logger.info('self submission detected') self.needs_release_manager = True return True src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev) package = target_package origin = self.lookup.get(target_project, package) origin_same = True if origin: origin_same = self._check_same_origin(origin, src_project) if src_srcinfo is None: # source package does not exist? # handle here to avoid crashing on the next line self.logger.warn("Could not get source info for %s/%s@%s" % (src_project, src_package, src_rev)) return False if self.ibs and target_project.startswith('SUSE:SLE'): review_result = None prj = 'openSUSE.org:openSUSE:Factory' if self.is_package_in_project(prj, package): # True or None (open request) are acceptable for SLE. in_factory = self._check_factory(package, src_srcinfo, prj) if in_factory: review_result = True self.source_in_factory = True elif in_factory is None: self.pending_factory_submission = True else: self.logger.info('different sources in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) else: self.logger.info( 'the package is not in Factory, nor submitted there') if review_result == None: other_projects_to_check = [] m = re.match('SUSE:SLE-(\d+)(?:-SP(\d+)):', target_project) if m: sle_version = int(m.group(1)) sp_version = int(m.group(2)) versions_to_check = [] # yeah, too much harcoding here if sle_version == 12: versions_to_check = ['42.3'] elif sle_version == 15: versions_to_check = [ '15.%d' % i for i in range(sp_version + 1) ] else: self.logger.error("can't handle %d.%d", sle_version, sp_version) for version in versions_to_check: leap = 'openSUSE.org:openSUSE:Leap:%s' % (version) other_projects_to_check += [ leap, leap + ':Update', leap + ':NonFree', leap + ':NonFree:Update' ] for prj in other_projects_to_check: if self.is_package_in_project(prj, package): self.logger.info('checking {}'.format(prj)) if self._check_factory(package, src_srcinfo, prj) is True: self.logger.info( 'found source match in {}'.format(prj)) else: self.logger.info('different sources in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) devel_project, devel_package = devel_project_get( self.apiurl, 'openSUSE.org:openSUSE:Factory', package) if devel_project is not None: # specifying devel package is optional if devel_package is None: devel_package = package if self.is_package_in_project(devel_project, devel_package): if self._check_matching_srcmd5( devel_project, devel_package, src_srcinfo.verifymd5) == True: self.logger.info( 'matching sources in {}/{}'.format( devel_project, devel_package)) return True else: self.logger.info( 'different sources in devel project {}'.format( self.rdiff_link(src_project, src_package, src_rev, devel_project, devel_package))) else: self.logger.info('no devel project found for {}/{}'.format( 'openSUSE.org:openSUSE:Factory', package)) self.logger.info( 'no matching sources found anywhere. Needs a human to decide whether that is ok. Please provide some justification to help that person.' ) if not review_result: review_result = origin_same if origin_same: self.logger.info("ok, origin %s unchanged", origin) else: # only log origin state if it's taken into consideration for the review result self.logger.info( "Submitted from a different origin than expected ('%s')", origin) if not review_result and self.override_allow: # Rather than decline, leave review open in-case of change and # ask release manager for input via override comment. self.logger.info( 'Comment `(at){} override accept` to force accept.'.format( self.review_user)) self.needs_release_manager = True review_result = None return review_result if target_project.endswith(':Update'): self.logger.info("expected origin is '%s' (%s)", origin, "unchanged" if origin_same else "changed") # Only when not from current product should request require maintainer review. self.do_check_maintainer_review = False if origin_same: return True good = self._check_matching_srcmd5(origin, target_package, src_srcinfo.verifymd5) if good: self.logger.info( 'submission source found in origin ({})'.format(origin)) return good good = self.factory._check_requests(origin, target_package, src_srcinfo.verifymd5) if good or good == None: self.logger.info( 'found pending submission against origin ({})'.format( origin)) return good # TODO #1662: Uncomment once maintbot has been superseded and leaper # is no longer run in comment-only mode. #self.do_check_maintainer_review = True return None elif self.action.type == 'maintenance_incident': self.logger.debug( 'unhandled incident pattern (targetting non :Update project)') return True # obviously if src_project in ('openSUSE:Factory', 'openSUSE:Factory:NonFree'): self.source_in_factory = True is_fine_if_factory = False not_in_factory_okish = False if origin: self.logger.info("expected origin is '%s' (%s)", origin, "unchanged" if origin_same else "changed") if origin.startswith('Devel;'): if origin_same == False: self.logger.debug("not submitted from devel project") return False is_fine_if_factory = True not_in_factory_okish = True if self.must_approve_version_updates: self.needs_release_manager = True # fall through to check history and requests elif origin.startswith('openSUSE:Factory'): # A large number of requests are created by hand that leaper # would have created via update_crawler.py. This applies to # other origins, but primary looking to let Factory submitters # know that there is no need to make manual submissions to both. # Since it has a lookup entry it is not a new package. self.automatic_submission = False if self.must_approve_version_updates: self.needs_release_manager = True if origin == src_project: self.source_in_factory = True # no need to approve submissions from Factory if # the lookup file points to Factory. Just causes # spam for many maintainers #1393 self.do_check_maintainer_review = False is_fine_if_factory = True # fall through to check history and requests elif origin == 'FORK': is_fine_if_factory = True if not src_project.startswith('SUSE:SLE-'): not_in_factory_okish = True self.needs_check_source = True self.needs_release_manager = True # fall through to check history and requests # TODO Ugly save for 15.1 (n-1). elif origin.startswith('openSUSE:Leap:15.0'): if self.must_approve_maintenance_updates: self.needs_release_manager = True if src_project.startswith('openSUSE:Leap'): self.do_check_maintainer_review = False # submitted from :Update if origin_same: self.logger.debug("submission from 15.0 ok") return True # switching to sle package might make sense if src_project.startswith('SUSE:SLE-15'): self.needs_release_manager = True self.do_check_maintainer_review = False return True # submitted from elsewhere but is in :Update else: good = self._check_matching_srcmd5( 'openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5) if good: self.logger.info("submission found in 15.0") return good # check release requests too good = self.factory._check_requests( 'openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5) if good or good == None: self.logger.debug("found request") return good # let's see where it came from before oldorigin = self.lookup.get('openSUSE:Leap:15.0', target_package) if oldorigin: self.logger.debug("oldorigin {}".format(oldorigin)) # Factory. So it's ok to keep upgrading it to Factory # TODO: whitelist packages where this is ok and block others? self.logger.info("Package was from %s in 15.0", oldorigin) if oldorigin.startswith('openSUSE:Factory'): # check if an attempt to switch to SLE package is made for sp in ('SP1:GA', 'SP1:Update'): good = self._check_matching_srcmd5( 'SUSE:SLE-15-{}'.format(sp), target_package, src_srcinfo.verifymd5) if good: self.logger.info( "request sources come from SLE") self.needs_release_manager = True return good # TODO Ugly save for 15.2 (n-2). elif False and oldorigin.startswith('openSUSE:Leap:15.0'): self.logger.info("Package was from %s in 15.0", oldorigin) # the release manager needs to review attempts to upgrade to Factory is_fine_if_factory = True self.needs_release_manager = True elif origin.startswith('SUSE:SLE-15'): if self.must_approve_maintenance_updates: self.needs_release_manager = True if src_project.startswith('SUSE:SLE-15'): self.do_check_maintainer_review = False for v in ('15.0', '15.1'): prj = 'openSUSE:Leap:{}:SLE-workarounds'.format(v) if self.is_package_in_project(prj, target_package): self.logger.info("found package in %s", prj) if not self._check_matching_srcmd5( prj, target_package, src_srcinfo.verifymd5): self.logger.info( "sources in %s are NOT identical", self.rdiff_link(src_project, src_package, src_rev, prj, package)) self.needs_release_manager = True # submitted from :Update if origin == src_project: self.logger.debug("submission origin ok") return True elif origin.endswith(':GA') \ and src_project == origin[:-2]+'Update': self.logger.debug("sle update submission") return True # check if submitted from higher SP priolist = [ 'SUSE:SLE-15:', 'SUSE:SLE-15-SP1:', 'SUSE:SLE-15-SP2:', 'SUSE:SLE-15-SP3:' ] for i in range(len(priolist) - 1): if origin.startswith(priolist[i]): for prj in priolist[i + 1:]: if src_project.startswith(prj): self.logger.info( "submission from higher service pack %s:* ok", prj) return True in_sle_origin = self._check_factory(target_package, src_srcinfo, origin) if in_sle_origin: self.logger.info( 'parallel submission, also in {}'.format(origin)) return True self.needs_release_manager = True # the release manager needs to review attempts to upgrade to Factory is_fine_if_factory = True else: self.logger.error("unhandled origin %s", origin) return False else: # no origin # submission from SLE is ok if src_project.startswith('SUSE:SLE-15'): self.do_check_maintainer_review = False return True # new package submitted from Factory. Check if it was in # 42.3 before and skip maintainer review if so. subprj = src_project[len('openSUSE:Factory'):] # disabled for reference. Needed again for 16.0 probably if False and self.source_in_factory and target_project.startswith('openSUSE:Leap:15.0') \ and self.is_package_in_project('openSUSE:Leap:42.3'+subprj, package): self.logger.info('package was in 42.3') self.do_check_maintainer_review = False return True is_fine_if_factory = True self.needs_release_manager = True if origin is None or not origin.startswith('SUSE:SLE-'): for p in (':Update', ':GA'): prj = 'SUSE:SLE-15' + p if self.is_package_in_project(prj, package): self.logger.info('Package is in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) break is_in_factory = self.source_in_factory # we came here because none of the above checks find it good, so # let's see if the package is in Factory at least if is_in_factory is None: is_in_factory = self._check_factory(package, src_srcinfo) if is_in_factory: self.source_in_factory = True self.needs_reviewteam = False self.needs_legal_review = False elif is_in_factory is None: self.pending_factory_submission = True self.needs_reviewteam = False self.needs_legal_review = False else: if src_project.startswith('SUSE:SLE-15') \ or src_project.startswith('openSUSE:Leap:15.'): self.needs_reviewteam = False self.needs_legal_review = False else: self.needs_reviewteam = True self.needs_legal_review = True self.source_in_factory = False if is_fine_if_factory: if self.source_in_factory: return True elif self.pending_factory_submission: return None elif not_in_factory_okish: self.needs_reviewteam = True self.needs_legal_review = True return True if self.override_allow: # Rather than decline, leave review open and ask release # manager for input via override comment. self.logger.info( 'Comment `(at){} override accept` to force accept.'.format( self.review_user)) self.needs_release_manager = True return None return False
def check_source_submission(self, src_project, src_package, src_rev, target_project, target_package): super(Leaper, self).check_source_submission(src_project, src_package, src_rev, target_project, target_package) self.automatic_submission = False if src_project == target_project and src_package == target_package: self.logger.info('self submission detected') self.needs_release_manager = True return True src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev) package = target_package origin = None if src_srcinfo is None: # source package does not exist? # handle here to avoid crashing on the next line self.logger.warn("Could not get source info for %s/%s@%s" % (src_project, src_package, src_rev)) return False if self.ibs and target_project.startswith('SUSE:SLE'): if package in self.lookup_sle15: origin = self.lookup_sle15[package] origin_same = True if origin: origin_same = self._check_same_origin(origin, src_project) self.logger.info("expected origin is '%s' (%s)", origin, "unchanged" if origin_same else "changed") prj = 'openSUSE.org:openSUSE:Factory' # True or None (open request) are acceptable for SLE. self.source_in_factory = self._check_factory( package, src_srcinfo, prj) if self.source_in_factory is None: self.pending_factory_submission = True if self.source_in_factory is not False: return self.source_in_factory # got false. could mean package doesn't exist or no match if self.is_package_in_project(prj, package): self.logger.info('different sources in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) prj = 'openSUSE.org:openSUSE:Leap:15.0' # TODO Ugly save for SLE-15-SP1. if False and self.is_package_in_project(prj, package): if self._check_factory(package, src_srcinfo, prj) is True: self.logger.info('found source match in {}'.format(prj)) else: self.logger.info('different sources in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) devel_project, devel_package = devel_project_get( self.apiurl, 'openSUSE.org:openSUSE:Factory', package) if devel_project is not None: # specifying devel package is optional if devel_package is None: devel_package = package if self.is_package_in_project(devel_project, devel_package): if self.factory._check_project( devel_project, devel_package, src_srcinfo.verifymd5) == True: self.logger.info('matching sources in {}/{}'.format( devel_project, devel_package)) return True else: self.logger.info('different sources in {}'.format( self.rdiff_link(src_project, src_package, src_rev, devel_project, devel_package))) else: self.logger.info('no devel project found for {}/{}'.format( 'openSUSE.org:openSUSE:Factory', package)) #self.logger.info('no matching sources in Factory, Leap:15.0, nor devel project') self.logger.info( 'no matching sources in Factory, nor devel project') if origin_same is False: # Rather than decline, leave review open in-case of change and # ask release manager for input via override comment. self.logger.info( 'Comment `(at){} override accept` to force accept.'.format( self.review_user)) self.needs_release_manager = True return None return origin_same if package in self.lookup_150: origin = self.lookup_150[package] # obviously if src_project in ('openSUSE:Factory', 'openSUSE:Factory:NonFree'): self.source_in_factory = True is_fine_if_factory = False not_in_factory_okish = False if origin: origin_same = self._check_same_origin(origin, src_project) self.logger.info("expected origin is '%s' (%s)", origin, "unchanged" if origin_same else "changed") if origin.startswith('Devel;'): if origin_same == False: self.logger.debug("not submitted from devel project") return False is_fine_if_factory = True not_in_factory_okish = True if self.must_approve_version_updates: self.needs_release_manager = True # fall through to check history and requests elif origin.startswith('openSUSE:Factory'): # A large number of requests are created by hand that leaper # would have created via update_crawler.py. This applies to # other origins, but primary looking to let Factory submitters # know that there is no need to make manual submissions to both. # Since it has a lookup entry it is not a new package. self.automatic_submission = False if self.must_approve_version_updates: self.needs_release_manager = True if origin == src_project: self.source_in_factory = True # no need to approve submissions from Factory if # the lookup file points to Factory. Just causes # spam for many maintainers #1393 self.do_check_maintainer_review = False is_fine_if_factory = True # fall through to check history and requests elif origin == 'FORK': is_fine_if_factory = True if not src_project.startswith('SUSE:SLE-'): not_in_factory_okish = True self.needs_check_source = True self.needs_release_manager = True # fall through to check history and requests # TODO Ugly save for 15.1 (n-1). elif False and origin.startswith('openSUSE:Leap:15.0'): if self.must_approve_maintenance_updates: self.needs_release_manager = True # submitted from :Update if origin_same: self.logger.debug("submission from 15.0 ok") return True # switching to sle package might make sense if src_project.startswith('SUSE:SLE-15'): self.needs_release_manager = True return True # submitted from elsewhere but is in :Update else: good = self.factory._check_project( 'openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5) if good: self.logger.info("submission found in 15.0") return good # check release requests too good = self.factory._check_requests( 'openSUSE:Leap:15.0:Update', target_package, src_srcinfo.verifymd5) if good or good == None: self.logger.debug("found request") return good # let's see where it came from before if package in self.lookup_150: oldorigin = self.lookup_150[package] self.logger.debug("oldorigin {}".format(oldorigin)) # Factory. So it's ok to keep upgrading it to Factory # TODO: whitelist packages where this is ok and block others? self.logger.info("Package was from %s in 15.0", oldorigin) if oldorigin.startswith('openSUSE:Factory'): # check if an attempt to switch to SLE package is made for sp in ('SP1:GA', 'SP1:Update'): good = self.factory._check_project( 'SUSE:SLE-15-{}'.format(sp), target_package, src_srcinfo.verifymd5) if good: self.logger.info( "request sources come from SLE") self.needs_release_manager = True return good # TODO Ugly save for 15.2 (n-2). elif False and oldorigin.startswith('openSUSE:Leap:15.0'): o = self.lookup_150[package] self.logger.info("Package was from %s in 15.0", o) # the release manager needs to review attempts to upgrade to Factory is_fine_if_factory = True self.needs_release_manager = True elif origin.startswith('SUSE:SLE-15'): if self.must_approve_maintenance_updates: self.needs_release_manager = True for v in ('15.0', ): prj = 'openSUSE:Leap:{}:SLE-workarounds'.format(v) if self.is_package_in_project(prj, target_package): self.logger.info("found package in %s", prj) if not self.factory._check_project( prj, target_package, src_srcinfo.verifymd5): self.logger.info( "sources in %s are [NOT identical](%s)", prj, self.rdiff_link(src_project, src_package, src_rev, prj, package)) self.needs_release_manager = True # submitted from :Update if origin == src_project: self.logger.debug("submission origin ok") return True elif origin.endswith(':GA') \ and src_project == origin[:-2]+'Update': self.logger.debug("sle update submission") return True # check if submitted from higher SP priolist = [ 'SUSE:SLE-15:', 'SUSE:SLE-15-SP1:', 'SUSE:SLE-15-SP2:', 'SUSE:SLE-15-SP3:' ] for i in range(len(priolist) - 1): if origin.startswith(priolist[i]): for prj in priolist[i + 1:]: if src_project.startswith(prj): self.logger.info( "submission from higher service pack %s:* ok", prj) return True in_sle_origin = self._check_factory(target_package, src_srcinfo, origin) if in_sle_origin: self.logger.info( 'parallel submission, also in {}'.format(origin)) return True self.needs_release_manager = True # the release manager needs to review attempts to upgrade to Factory is_fine_if_factory = True else: self.logger.error("unhandled origin %s", origin) return False else: # no origin # submission from SLE is ok if src_project.startswith('SUSE:SLE-15'): self.do_check_maintainer_review = False return True # new package submitted from Factory. Check if it was in # 42.3 before and skip maintainer review if so. subprj = src_project[len('openSUSE:Factory'):] if self.source_in_factory and target_project.startswith('openSUSE:Leap:15.0') \ and self.is_package_in_project('openSUSE:Leap:42.3'+subprj, package): self.logger.info('package was in 42.3') self.do_check_maintainer_review = False return True is_fine_if_factory = True self.needs_release_manager = True if origin is None or not origin.startswith('SUSE:SLE-'): for p in (':Update', ':GA'): prj = 'SUSE:SLE-15' + p if self.is_package_in_project(prj, package): self.logger.info('Package is in {}'.format( self.rdiff_link(src_project, src_package, src_rev, prj, package))) break is_in_factory = self.source_in_factory # we came here because none of the above checks find it good, so # let's see if the package is in Factory at least if is_in_factory is None: is_in_factory = self._check_factory(package, src_srcinfo) if is_in_factory: self.source_in_factory = True self.needs_reviewteam = False self.needs_legal_review = False elif is_in_factory is None: self.pending_factory_submission = True self.needs_reviewteam = False self.needs_legal_review = False else: if src_project.startswith('SUSE:SLE-15') \ or src_project.startswith('openSUSE:Leap:15.'): self.needs_reviewteam = False self.needs_legal_review = False else: self.needs_reviewteam = True self.needs_legal_review = True self.source_in_factory = False if is_fine_if_factory: if self.source_in_factory: return True elif self.pending_factory_submission: return None elif not_in_factory_okish: self.needs_reviewteam = True self.needs_legal_review = True return True if self.override_allow: # Rather than decline, leave review open and ask release # manager for input via override comment. self.logger.info( 'Comment `(at){} override accept` to force accept.'.format( self.review_user)) self.needs_release_manager = True return None return False
def crawl(self): """Main method""" succeeded_packages = [] succeeded_packages = self.get_build_succeeded_packages(self.from_prj) if not len(succeeded_packages) > 0: logging.info('No build succeeded package in %s'%self.from_prj) return # randomize the list random.shuffle(succeeded_packages) # get souce packages from target target_packages = self.get_source_packages(self.to_prj) deleted_packages = self.get_deleted_packages(self.to_prj) if self.to_prj.startswith("openSUSE:"): deleted_packages = deleted_packages + self.get_deleted_packages(OPENSUSE_PREVERSION) pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, 'openSUSE:Factory') skip_pkgs_list = self.load_skip_pkgs_list(pseudometa_project, pseudometa_package).splitlines() ms_packages = [] # collect multi specs packages for i in range(0, min(int(self.submit_limit), len(succeeded_packages))): package = succeeded_packages[i] submit_ok = True if package in deleted_packages: logging.info('%s has been dropped from %s, ignore it!'%(package, self.to_prj)) submit_ok = False if self.is_sle_base_pkgs(package) is True: logging.info('%s origin from SLE base, skip for now!'%package) submit_ok = False # make sure it is new package new_pkg = self.is_new_package(self.to_prj, package) if new_pkg is not True: logging.info('%s is not a new package, do not submit.' % package) submit_ok = False multi_specs = self.check_multiple_specfiles(self.factory, package) if multi_specs is None: logging.info('%s does not exist in %s'%(package, 'openSUSE:Factory')) submit_ok = False if multi_specs: if multi_specs['linkinfo']: logging.info('%s in %s is sub-package of %s, skip it!'%(package, 'openSUSE:Factory', multi_specs['linkinfo'])) ms_packages.append(package) submit_ok = False for spec in multi_specs['specs']: if spec not in succeeded_packages: logging.info('%s is sub-pacakge of %s but build failed, skip it!'%(spec, package)) submit_ok = False if not submit_ok: continue # make sure the package non-exist in target yet ie. expand=False if package not in target_packages: # make sure there is no request against same package request = self.get_request_list(package) if request: logging.debug("There is a request to %s / %s already or it has been declined/revoked, skip!"%(package, self.to_prj)) else: logging.info("%d - Preparing submit %s to %s"%(i, package, self.to_prj)) # get devel project devel_prj, devel_pkg = devel_project_get(self.apiurl, self.factory, package) # check devel project does not in the skip list if devel_prj in self.skip_devel_project_list: # check the except packages list match = None for elem in self.except_pkgs_list: m = re.search(elem, package) if m is not None: match = True if match is not True: logging.info('%s/%s is in the skip list, do not submit.' % (devel_prj, package)) continue else: pass # check package does not in the skip list match = None for elem in skip_pkgs_list: m = re.search(elem, package) if m is not None: match = True if match is True: logging.info('%s is in the skip list, do not submit.' % package) continue else: pass res = self.create_submitrequest(package) if res and res is not None: logging.info('Created request %s for %s' % (res, package)) else: logging.error('Error occurred when creating submit request') else: logging.debug('%s is exist in %s, skip!'%(package, self.to_prj)) # dump multi specs packages print("Multi-specfile packages:") if ms_packages: for pkg in ms_packages: print pkg else: print 'None'
def check_source_submission(self, source_project, source_package, source_revision, target_project, target_package): super(CheckSource, self).check_source_submission(source_project, source_package, source_revision, target_project, target_package) self.target_project_config(target_project) if not self.ignore_devel: self.logger.info( 'checking if target package exists and has devel project') devel_project, devel_package = devel_project_get( self.apiurl, target_project, target_package) if devel_project: if (source_project != devel_project or source_package != devel_package) and \ not(source_project == target_project and source_package == target_package): # Not from proper devel project/package and not self-submission. self.review_messages[ 'declined'] = 'Expected submission from devel package %s/%s' % ( devel_project, devel_package) return False else: # Check to see if other packages exist with the same source project # which indicates that the project has already been used as devel. if not self.is_devel_project(source_project, target_project): self.review_messages[ 'declined'] = '%s is not a devel project of %s, submit the package to a devel project first' % ( source_project, target_project) return False # Checkout and see if renaming package screws up version parsing. dir = os.path.expanduser('~/co/%s' % self.request.reqid) if os.path.exists(dir): self.logger.warn('directory %s already exists' % dir) shutil.rmtree(dir) os.makedirs(dir) os.chdir(dir) old_info = {'version': None} try: CheckSource.checkout_package(self.apiurl, target_project, target_package, pathname=dir, server_service_files=True, expand_link=True) shutil.rmtree(os.path.join(target_package, '.osc')) os.rename(target_package, '_old') old_info = self.package_source_parse(target_project, target_package) except urllib2.HTTPError: self.logger.error('failed to checkout %s/%s' % (target_project, target_package)) CheckSource.checkout_package(self.apiurl, source_project, source_package, revision=source_revision, pathname=dir, server_service_files=True, expand_link=True) os.rename(source_package, target_package) shutil.rmtree(os.path.join(target_package, '.osc')) new_info = self.package_source_parse(source_project, source_package, source_revision) if new_info['name'] != target_package: shutil.rmtree(dir) self.review_messages[ 'declined'] = "A package submitted as %s has to build as 'Name: %s' - found Name '%s'" % ( target_package, target_package, new_info['name']) return False # We want to see the same package name in the devel project as in the distro; anything else calls for confusion if source_package != target_package: self.review_messages[ 'declined'] = "No in-air renames: The package must be called the same in the devel project as in the target project" return False # Run check_source.pl script and interpret output. source_checker = os.path.join(CheckSource.SCRIPT_PATH, 'check_source.pl') civs = '' new_version = None if old_info['version'] and old_info['version'] != new_info['version']: new_version = new_info['version'] civs += "NEW_VERSION='{}' ".format(new_version) civs += 'LC_ALL=C perl %s _old %s 2>&1' % (source_checker, target_package) p = subprocess.Popen(civs, shell=True, stdout=subprocess.PIPE, close_fds=True) ret = os.waitpid(p.pid, 0)[1] checked = p.stdout.readlines() output = ' '.join(checked).translate(None, '\033') os.chdir('/tmp') # ret = 0 : Good # ret = 1 : Bad # ret = 2 : Bad but can be non-fatal in some cases if ret > 1 and target_project.startswith('openSUSE:Leap:') and ( source_project.startswith('SUSE:SLE-15:') or source_project.startswith('openSUSE:Factory')): pass elif ret != 0: shutil.rmtree(dir) self.review_messages[ 'declined'] = "Output of check script:\n" + output return False shutil.rmtree(dir) self.review_messages['accepted'] = 'Check script succeeded' if len(checked): self.review_messages[ 'accepted'] += "\n\nOutput of check script (non-fatal):\n" + output if not self.skip_add_reviews: if self.review_team is not None: self.add_review(self.request, by_group=self.review_team, msg='Please review sources') if self.only_changes(): self.logger.debug('only .changes modifications') staging_group = self.staging_group(target_project) if staging_group and not self.dryrun: osc.core.change_review_state( self.apiurl, str(self.request.reqid), 'accepted', by_group=staging_group, message= 'skipping the staging process since only .changes modifications' ) elif self.repo_checker is not None: self.add_review(self.request, by_user=self.repo_checker, msg='Please review build success') return True
def devel_workflow(self, only_devel): self.remote_config_set_age_minimum() devel_project = self.randomString('devel') package = self.randomString('package') request = self.wf.create_submit_request(devel_project, package) attribute_value_save(self.wf.apiurl, devel_project, 'ApprovedRequestSource', '', 'OBS') if not only_devel: self.assertReviewBot(request.reqid, self.bot_user, 'new', 'new') comment = [ '<!-- OriginManager state=seen result=None -->', 'Source not found in allowed origins:', f'- {self.product_project}', f'Decision may be overridden via `@{self.bot_user} override`.', ] self.assertComment(request.reqid, comment) CommentAPI(self.wf.api.apiurl).add_comment( request_id=request.reqid, comment=f'@{self.bot_user} change_devel') comment = 'change_devel command by {}'.format('Admin') else: comment = 'only devel origin allowed' self.assertReviewBot(request.reqid, self.bot_user, 'new', 'accepted') self.assertAnnotation(request.reqid, { 'comment': comment, 'origin': devel_project, }) request.change_state('accepted') memoize_session_reset() self.osc_user(self.bot_user) request_future = origin_update(self.wf.apiurl, self.wf.project, package) self.assertNotEqual(request_future, False) if request_future: request_id_change_devel = request_future.print_and_create() # Ensure a second request is not triggered. request_future = origin_update(self.wf.apiurl, self.wf.project, package) self.assertEqual(request_future, False) self.osc_user_pop() memoize_session_reset() origin_info = origin_find(self.wf.apiurl, self.wf.project, package) self.assertEqual(origin_info, None) self.assertReviewBot(request_id_change_devel, self.bot_user, 'new', 'accepted') self.assertAnnotation(request_id_change_devel, { 'origin': devel_project, }) # Origin should change before request is accepted since it is properly # annotated and without fallback review. memoize_session_reset() origin_info = origin_find(self.wf.apiurl, self.wf.project, package) self.assertEqual(str(origin_info), devel_project) self.wf.projects[devel_project].packages[0].create_commit() self.osc_user(self.bot_user) request_future = origin_update(self.wf.apiurl, self.wf.project, package) self.assertNotEqual(request_future, False) if request_future: request_id_update = request_future.print_and_create() request_future = origin_update(self.wf.apiurl, self.wf.project, package) self.assertEqual(request_future, False) self.osc_user_pop() self.assertReviewBot(request_id_update, self.bot_user, 'new', 'accepted') self.assertAnnotation(request_id_update, { 'origin': devel_project, }) memoize_session_reset() devel_project_actual, _ = devel_project_get(self.wf.apiurl, self.wf.project, package) self.assertEqual(devel_project_actual, None) request = get_request(self.wf.apiurl, request_id_change_devel) request_state_change(self.wf.apiurl, request_id_change_devel, 'accepted') memoize_session_reset() devel_project_actual, devel_package_actual = devel_project_get( self.wf.apiurl, self.wf.project, package) self.assertEqual(devel_project_actual, devel_project) self.assertEqual(devel_package_actual, package) request = get_request(self.wf.apiurl, request_id_update) request_state_change(self.wf.apiurl, request_id_update, 'accepted') devel_project_new = self.randomString('develnew') self.wf.create_package(devel_project_new, package) attribute_value_save(self.wf.apiurl, devel_project_new, 'ApprovedRequestSource', '', 'OBS') copy_package(self.wf.apiurl, devel_project, package, self.wf.apiurl, devel_project_new, package) request_future = request_create_change_devel( self.wf.apiurl, devel_project_new, package, self.wf.project) self.assertNotEqual(request_future, False) if request_future: request_id_change_devel_new = request_future.print_and_create() self.assertReviewBot(request_id_change_devel_new, self.bot_user, 'new', 'accepted') self.assertAnnotation(request_id_change_devel_new, { 'origin': devel_project_new, 'origin_old': devel_project, }) self.accept_fallback_review(request_id_change_devel_new) request_state_change(self.wf.apiurl, request_id_change_devel_new, 'accepted') memoize_session_reset() origin_info = origin_find(self.wf.apiurl, self.wf.project, package) self.assertEqual(str(origin_info), devel_project_new)
def check_source_submission(self, source_project, source_package, source_revision, target_project, target_package): super(CheckSource, self).check_source_submission(source_project, source_package, source_revision, target_project, target_package) self.target_project_config(target_project) if self.single_action_require and len(self.request.actions) != 1: self.review_messages[ 'declined'] = 'Only one action per request allowed' return False kind = package_kind(self.apiurl, target_project, target_package) if kind == 'meta': self.review_messages[ 'accepted'] = 'Skipping all checks for meta packages' return True elif (kind is not None and kind != 'source'): self.review_messages[ 'declined'] = 'May not modify a non-source package of type {}'.format( kind) return False inair_renamed = target_package != source_package if not self.ignore_devel: self.logger.info( 'checking if target package exists and has devel project') devel_project, devel_package = devel_project_get( self.apiurl, target_project, target_package) if devel_project: if (source_project != devel_project or source_package != devel_package) and \ not(source_project == target_project and source_package == target_package): # Not from proper devel project/package and not self-submission. self.review_messages[ 'declined'] = 'Expected submission from devel package %s/%s' % ( devel_project, devel_package) return False else: # Check to see if other packages exist with the same source project # which indicates that the project has already been used as devel. if not self.is_devel_project(source_project, target_project): self.review_messages['declined'] = ( '%s is not a devel project of %s, submit the package to a devel project first. ' 'See https://en.opensuse.org/openSUSE:How_to_contribute_to_Factory#How_to_request_a_new_devel_project for details.' ) % (source_project, target_project) return False else: if source_project.endswith(':Update'): # Allow for submission like: # - source: openSUSE:Leap:15.0:Update/google-compute-engine.8258 # - target: openSUSE:Leap:15.1/google-compute-engine # Note: home:jberry:Update would also be allowed via this condition, # but that should be handled by leaper and human review. # Ignore a dot in package name (ex. tpm2.0-abrmd) and instead # only look for ending in dot number. match = re.match(r'(.*)\.\d+$', source_package) if match: inair_renamed = target_package != match.group(1) if not self.in_air_rename_allow and inair_renamed: self.review_messages[ 'declined'] = 'Source and target package names must match' return False # Checkout and see if renaming package screws up version parsing. dir = os.path.expanduser('~/co/%s' % self.request.reqid) if os.path.exists(dir): self.logger.warning('directory %s already exists' % dir) shutil.rmtree(dir) os.makedirs(dir) os.chdir(dir) old_info = {'version': None} try: CheckSource.checkout_package(self.apiurl, target_project, target_package, pathname=dir, server_service_files=True, expand_link=True) shutil.rmtree(os.path.join(target_package, '.osc')) os.rename(target_package, '_old') old_info = self.package_source_parse(target_project, target_package) except HTTPError as e: if e.code == 404: self.logger.info('target package does not exist %s/%s' % (target_project, target_package)) else: raise e CheckSource.checkout_package(self.apiurl, source_project, source_package, revision=source_revision, pathname=dir, server_service_files=True, expand_link=True) os.rename(source_package, target_package) shutil.rmtree(os.path.join(target_package, '.osc')) new_info = self.package_source_parse(source_project, source_package, source_revision, target_package) filename = new_info.get('filename', '') if not (filename.endswith('.kiwi') or filename == 'Dockerfile') and new_info['name'] != target_package: shutil.rmtree(dir) self.review_messages[ 'declined'] = "A package submitted as %s has to build as 'Name: %s' - found Name '%s'" % ( target_package, target_package, new_info['name']) return False # Run check_source.pl script and interpret output. source_checker = os.path.join(CheckSource.SCRIPT_PATH, 'check_source.pl') civs = '' new_version = None if old_info['version'] and old_info['version'] != new_info['version']: new_version = new_info['version'] civs += "NEW_VERSION='{}' ".format(new_version) civs += 'LC_ALL=C perl %s _old %s 2>&1' % (source_checker, target_package) p = subprocess.Popen(civs, shell=True, stdout=subprocess.PIPE, close_fds=True) ret = os.waitpid(p.pid, 0)[1] checked = decode_list(p.stdout.readlines()) output = ' '.join(checked).replace('\033', '') os.chdir('/tmp') # ret = 0 : Good # ret = 1 : Bad # ret = 2 : Bad but can be non-fatal in some cases if ret > 1 and target_project.startswith('openSUSE:Leap:') and ( source_project.startswith('SUSE:SLE-15:') or source_project.startswith('openSUSE:Factory')): pass elif ret != 0: shutil.rmtree(dir) self.review_messages[ 'declined'] = "Output of check script:\n" + output return False shutil.rmtree(dir) self.review_messages['accepted'] = 'Check script succeeded' if len(checked): self.review_messages[ 'accepted'] += "\n\nOutput of check script (non-fatal):\n" + output if not self.skip_add_reviews: if self.add_review_team and self.review_team is not None: self.add_review(self.request, by_group=self.review_team, msg='Please review sources') if self.only_changes(): self.logger.debug('only .changes modifications') if self.staging_group and self.review_user in group_members( self.apiurl, self.staging_group): if not self.dryrun: osc.core.change_review_state( self.apiurl, str(self.request.reqid), 'accepted', by_group=self.staging_group, message= 'skipping the staging process since only .changes modifications' ) else: self.logger.debug( 'unable to skip staging review since not a member of staging group' ) elif self.repo_checker is not None: self.add_review(self.request, by_user=self.repo_checker, msg='Please review build success') if self.bad_rpmlint_entries: if self.has_whitelist_warnings(source_project, source_package, target_project, target_package): # if there are any add a review for the security team # maybe add the found warnings to the message for the review self.add_review(self.request, by_group=self.security_review_team, msg=CheckSource.AUDIT_BUG_MESSAGE) if self.suppresses_whitelist_warnings(source_project, source_package): self.add_review(self.request, by_group=self.security_review_team, msg=CheckSource.AUDIT_BUG_MESSAGE) return True
def check_source_submission(self, source_project, source_package, source_revision, target_project, target_package): super(CheckSource, self).check_source_submission(source_project, source_package, source_revision, target_project, target_package) self.target_project_config(target_project) if self.single_action_require and len(self.request.actions) != 1: self.review_messages['declined'] = 'Only one action per request allowed' return False if target_package.startswith('00') or target_package.startswith('_'): self.review_messages['accepted'] = 'Skipping all checks for product related packages' return True inair_renamed = target_package != source_package if not self.ignore_devel: self.logger.info('checking if target package exists and has devel project') devel_project, devel_package = devel_project_get(self.apiurl, target_project, target_package) if devel_project: if (source_project != devel_project or source_package != devel_package) and \ not(source_project == target_project and source_package == target_package): # Not from proper devel project/package and not self-submission. self.review_messages['declined'] = 'Expected submission from devel package %s/%s' % (devel_project, devel_package) return False else: # Check to see if other packages exist with the same source project # which indicates that the project has already been used as devel. if not self.is_devel_project(source_project, target_project): self.review_messages['declined'] = ( '%s is not a devel project of %s, submit the package to a devel project first. ' 'See https://en.opensuse.org/openSUSE:How_to_contribute_to_Factory#How_to_request_a_new_devel_project for details.' ) % (source_project, target_project) return False else: if source_project.endswith(':Update'): # Allow for submission like: # - source: openSUSE:Leap:15.0:Update/google-compute-engine.8258 # - target: openSUSE:Leap:15.1/google-compute-engine # Note: home:jberry:Update would also be allowed via this condition, # but that should be handled by leaper and human review. # Ignore a dot in package name (ex. tpm2.0-abrmd) and instead # only look for ending in dot number. match = re.match(r'(.*)\.\d+$', source_package) if match: inair_renamed = target_package != match.group(1) if not self.in_air_rename_allow and inair_renamed: self.review_messages['declined'] = 'Source and target package names must match' return False # Checkout and see if renaming package screws up version parsing. dir = os.path.expanduser('~/co/%s' % self.request.reqid) if os.path.exists(dir): self.logger.warn('directory %s already exists' % dir) shutil.rmtree(dir) os.makedirs(dir) os.chdir(dir) old_info = {'version': None} try: CheckSource.checkout_package(self.apiurl, target_project, target_package, pathname=dir, server_service_files=True, expand_link=True) shutil.rmtree(os.path.join(target_package, '.osc')) os.rename(target_package, '_old') old_info = self.package_source_parse(target_project, target_package) except HTTPError: self.logger.error('failed to checkout %s/%s' % (target_project, target_package)) CheckSource.checkout_package(self.apiurl, source_project, source_package, revision=source_revision, pathname=dir, server_service_files=True, expand_link=True) os.rename(source_package, target_package) shutil.rmtree(os.path.join(target_package, '.osc')) new_info = self.package_source_parse(source_project, source_package, source_revision) if new_info['name'] != target_package: shutil.rmtree(dir) self.review_messages['declined'] = "A package submitted as %s has to build as 'Name: %s' - found Name '%s'" % (target_package, target_package, new_info['name']) return False # Run check_source.pl script and interpret output. source_checker = os.path.join(CheckSource.SCRIPT_PATH, 'check_source.pl') civs = '' new_version = None if old_info['version'] and old_info['version'] != new_info['version']: new_version = new_info['version'] civs += "NEW_VERSION='{}' ".format(new_version) civs += 'LC_ALL=C perl %s _old %s 2>&1' % (source_checker, target_package) p = subprocess.Popen(civs, shell=True, stdout=subprocess.PIPE, close_fds=True) ret = os.waitpid(p.pid, 0)[1] checked = p.stdout.readlines() output = ' '.join(checked).translate(None, '\033') os.chdir('/tmp') # ret = 0 : Good # ret = 1 : Bad # ret = 2 : Bad but can be non-fatal in some cases if ret > 1 and target_project.startswith('openSUSE:Leap:') and (source_project.startswith('SUSE:SLE-15:') or source_project.startswith('openSUSE:Factory')): pass elif ret != 0: shutil.rmtree(dir) self.review_messages['declined'] = "Output of check script:\n" + output return False shutil.rmtree(dir) self.review_messages['accepted'] = 'Check script succeeded' if len(checked): self.review_messages['accepted'] += "\n\nOutput of check script (non-fatal):\n" + output if not self.skip_add_reviews: if self.add_review_team and self.review_team is not None: self.add_review(self.request, by_group=self.review_team, msg='Please review sources') if self.only_changes(): self.logger.debug('only .changes modifications') if self.staging_group and self.review_user in group_members(self.apiurl, self.staging_group): if not self.dryrun: osc.core.change_review_state(self.apiurl, str(self.request.reqid), 'accepted', by_group=self.staging_group, message='skipping the staging process since only .changes modifications') else: self.logger.debug('unable to skip staging review since not a member of staging group') elif self.repo_checker is not None: self.add_review(self.request, by_user=self.repo_checker, msg='Please review build success') return True