class ToTestBase(object): """Base class to store the basic interface""" def __init__(self, project, dryrun = False): self.project = project self.dryrun = dryrun self.api = StagingAPI(osc.conf.config['apiurl'], project='openSUSE:%s' % project) self.known_failures = self.known_failures_from_dashboard(project) def openqa_group(self): return self.project def iso_prefix(self): return self.project def jobs_num(self): return 70 def current_version(self): return self.release_version() def binaries_of_product(self, project, product): url = self.api.makeurl(['build', project, 'images', 'local', product]) try: f = self.api.retried_GET(url) except urllib2.HTTPError: return [] ret = [] root = ET.parse(f).getroot() for binary in root.findall('binary'): ret.append(binary.get('filename')) return ret def get_current_snapshot(self): """Return the current snapshot in :ToTest""" # for now we hardcode all kind of things for binary in self.binaries_of_product('openSUSE:%s:ToTest' % self.project, '_product:openSUSE-cd-mini-%s' % self.arch()): result = re.match(r'openSUSE-%s-NET-.*-Snapshot(.*)-Media.iso' % self.iso_prefix(), binary) if result: return result.group(1) return None def ftp_build_version(self, tree): for binary in self.binaries_of_product('openSUSE:%s' % self.project, tree): result = re.match(r'openSUSE.*Build(.*)-Media1.report', binary) if result: return result.group(1) raise Exception("can't find %s version" % self.project) def release_version(self): url = self.api.makeurl(['build', 'openSUSE:%s' % self.project, 'standard', self.arch(), '_product:openSUSE-release']) f = self.api.retried_GET(url) root = ET.parse(f).getroot() for binary in root.findall('binary'): binary = binary.get('filename', '') result = re.match(r'.*-([^-]*)-[^-]*.src.rpm', binary) if result: return result.group(1) raise Exception("can't find %s version" % self.project) def find_openqa_results(self, snapshot): """Return the openqa jobs of a given snapshot and filter out the cloned jobs """ url = makeurl('https://openqa.opensuse.org', ['api', 'v1', 'jobs'], { 'group': self.openqa_group(), 'build': snapshot } ) f = self.api.retried_GET(url) jobs = [] for job in json.load(f)['jobs']: if job['clone_id'] or job['result'] == 'obsoleted': continue job['name'] = job['name'].replace(snapshot, '') jobs.append(job) return jobs def _result2str(self, result): if result == QA_INPROGRESS: return 'inprogress' elif result == QA_FAILED: return 'failed' else: return 'passed' def find_failed_module(self, testmodules): # print json.dumps(testmodules, sort_keys=True, indent=4) for module in testmodules: if module['result'] != 'failed': continue flags = module['flags'] if 'fatal' in flags or 'important' in flags: return module['name'] break logger.info('%s %s %s'%(module['name'], module['result'], module['flags'])) def overall_result(self, snapshot): """Analyze the openQA jobs of a given snapshot Returns a QAResult""" if snapshot is None: return QA_FAILED jobs = self.find_openqa_results(snapshot) if len(jobs) < self.jobs_num(): # not yet scheduled logger.warning('we have only %s jobs' % len(jobs)) return QA_INPROGRESS number_of_fails = 0 in_progress = False machines = [] for job in jobs: machines.append(job['settings']['MACHINE']) # print json.dumps(job, sort_keys=True, indent=4) if job['result'] in ('failed', 'incomplete', 'skipped', 'user_cancelled', 'obsoleted'): jobname = job['name'] + '@' + job['settings']['MACHINE'] # Record machines we have tests for if jobname in self.known_failures: self.known_failures.remove(jobname) continue number_of_fails += 1 # print json.dumps(job, sort_keys=True, indent=4), jobname failedmodule = self.find_failed_module(job['modules']) url = 'https://openqa.opensuse.org/tests/%s' % job['id'] print (jobname, url, failedmodule, job['retry_avbl']) # if number_of_fails < 3: continue elif job['result'] == 'passed' or job['result'] == 'softfailed': continue elif job['result'] == 'none': if job['state'] != 'cancelled': in_progress = True else: raise Exception(job['result']) if number_of_fails > 0: return QA_FAILED if in_progress: return QA_INPROGRESS machines = list(set(machines)) for item in machines: for item2 in self.known_failures: if item2.split('@')[1] == item: logger.info('now passing %s'%item2) return QA_PASSED def all_repos_done(self, project, codes=None): """Check the build result of the project and only return True if all repos of that project are either published or unpublished """ # coolo's experience says that 'finished' won't be # sufficient here, so don't try to add it :-) codes = ['published', 'unpublished'] if not codes else codes url = self.api.makeurl(['build', project, '_result'], {'code': 'failed'}) f = self.api.retried_GET(url) root = ET.parse(f).getroot() ready = True for repo in root.findall('result'): # ignore ports. 'factory' is used by arm for repos that are not # meant to use the totest manager. if repo.get('repository') in ('ports', 'factory', 'images_staging'): continue # ignore 32bit for now. We're only interesed in aarch64 here if repo.get('arch') in ('armv6l', 'armv7l'): continue if repo.get('dirty', '') == 'true': logger.info('%s %s %s -> %s'%(repo.get('project'), repo.get('repository'), repo.get('arch'), 'dirty')) ready = False if repo.get('code') not in codes: logger.info('%s %s %s -> %s'%(repo.get('project'), repo.get('repository'), repo.get('arch'), repo.get('code'))) ready = False return ready def maxsize_for_package(self, package): if re.match(r'.*-mini-.*', package): return 737280000 # a CD needs to match if re.match(r'.*-dvd5-.*', package): return 4700372992 # a DVD needs to match if re.match(r'.*-image-livecd-x11.*', package): return 681574400 # not a full CD if re.match(r'.*-image-livecd.*', package): return 999999999 # a GB stick if re.match(r'.*-dvd9-dvd-.*', package): return 8539996159 if package.startswith('_product:openSUSE-ftp-ftp-'): return None if package == '_product:openSUSE-Addon-NonOss-ftp-ftp-i586_x86_64': return None raise Exception('No maxsize for {}'.format(package)) def package_ok(self, project, package, repository, arch): """Checks one package in a project and returns True if it's succeeded """ query = {'package': package, 'repository': repository, 'arch': arch} url = self.api.makeurl(['build', project, '_result'], query) f = self.api.retried_GET(url) root = ET.parse(f).getroot() for repo in root.findall('result'): status = repo.find('status') if status.get('code') != 'succeeded': logger.info('%s %s %s %s -> %s'%(project, package, repository, arch, status.get('code'))) return False maxsize = self.maxsize_for_package(package) if not maxsize: return True url = self.api.makeurl(['build', project, repository, arch, package]) f = self.api.retried_GET(url) root = ET.parse(f).getroot() for binary in root.findall('binary'): if not binary.get('filename', '').endswith('.iso'): continue isosize = int(binary.get('size', 0)) if isosize > maxsize: logger.error('%s %s %s %s: %s'%(project, package, repository, arch, 'too large by %s bytes' % (isosize-maxsize))) return False return True def factory_snapshottable(self): """Check various conditions required for factory to be snapshotable """ if not self.all_repos_done('openSUSE:%s' % self.project): return False for product in self.ftp_products + self.main_products: if not self.package_ok('openSUSE:%s' % self.project, product, 'images', 'local'): return False if len(self.livecd_products): if not self.all_repos_done('openSUSE:%s:Live' % self.project): return False for arch in ['i586', 'x86_64' ]: for product in self.livecd_products: if not self.package_ok('openSUSE:%s:Live' % self.project, product, 'standard', arch): return False return True def release_package(self, project, package, set_release=None): query = {'cmd': 'release'} if set_release: query['setrelease'] = set_release # FIXME: make configurable. openSUSE:Factory:ARM currently has multiple # repos with release targets, so obs needs to know which one to release if project == 'openSUSE:Factory:ARM': query['repository'] = 'images' baseurl = ['source', project, package] url = self.api.makeurl(baseurl, query=query) if self.dryrun: logger.info("release %s/%s (%s)"%(project, package, set_release)) else: self.api.retried_POST(url) def update_totest(self, snapshot): logger.info('Updating snapshot %s' % snapshot) if not self.dryrun: self.api.switch_flag_in_prj('openSUSE:%s:ToTest' % self.project, flag='publish', state='disable') for product in self.ftp_products: self.release_package('openSUSE:%s' % self.project, product) for cd in self.livecd_products: self.release_package('openSUSE:%s:Live' % self.project, cd, set_release='Snapshot%s' % snapshot) for cd in self.main_products: self.release_package('openSUSE:%s' % self.project, cd, set_release='Snapshot%s' % snapshot) def publish_factory_totest(self): logger.info('Publish ToTest') if not self.dryrun: self.api.switch_flag_in_prj('openSUSE:%s:ToTest' % self.project, flag='publish', state='enable') def totest_is_publishing(self): """Find out if the publishing flag is set in totest's _meta""" url = self.api.makeurl(['source', 'openSUSE:%s:ToTest' % self.project, '_meta']) f = self.api.retried_GET(url) root = ET.parse(f).getroot() if not root.find('publish'): # default true return True for flag in root.find('publish'): if flag.get('repository', None) or flag.get('arch', None): continue if flag.tag == 'enable': return True return False def totest(self): current_snapshot = self.get_current_snapshot() new_snapshot = self.current_version() current_result = self.overall_result(current_snapshot) current_qa_version = self.api.load_file_content("%s:Staging" % self.api.project, "dashboard", "version_totest") logger.info('current_snapshot %s: %s'%(current_snapshot, self._result2str(current_result))) logger.debug('new_snapshot %s', new_snapshot) logger.debug('current_qa_version %s', current_qa_version) snapshotable = self.factory_snapshottable() logger.debug("snapshotable: %s", snapshotable) can_release = (current_result != QA_INPROGRESS and snapshotable) # not overwriting if new_snapshot == current_snapshot: logger.debug("no change in snapshot version") can_release = False elif not self.all_repos_done('openSUSE:%s:ToTest' % self.project): logger.debug("not all repos done, can't release") # the repos have to be done, otherwise we better not touch them with a new release can_release = False can_publish = (current_result == QA_PASSED) # already published if self.totest_is_publishing(): logger.debug("totest already publishing") can_publish = False if can_publish: if current_qa_version == current_snapshot: self.publish_factory_totest() self.write_version_to_dashboard("snapshot", current_snapshot) can_release = False # we have to wait else: # We reached a very bad status: openQA testing is 'done', but not of the same version # currently in :ToTest. This can happen when 'releasing' the product failed raise Exception("Publishing stopped: tested version (%s) does not match :ToTest version (%s)" % (current_qa_version, current_snapshot)) if can_release: self.update_totest(new_snapshot) self.write_version_to_dashboard("totest", new_snapshot) def release(self): new_snapshot = self.current_version() self.update_totest(new_snapshot) def known_failures_from_dashboard(self, project): known_failures = [] if self.project == "Factory:PowerPC": project = "Factory" else: project = self.project url = self.api.makeurl(['source', 'openSUSE:%s:Staging' % project, 'dashboard', 'known_failures']) f = self.api.retried_GET(url) for line in f: if not line[0] == '#': known_failures.append(line.strip()) return known_failures def write_version_to_dashboard(self, target, version): if not self.dryrun: url = self.api.makeurl(['source', 'openSUSE:%s:Staging' % self.project, 'dashboard', 'version_%s' % target]) osc.core.http_PUT(url + '?comment=Update+version', data=version)
def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects ${cmd_option_list} "accept" will accept all requests in $PROJECT:Staging:<LETTER> into $PROJECT If openSUSE:* project, requests marked ready from adi stagings will also be accepted. "acheck" will check if it is safe to accept new staging projects As $PROJECT is syncing the right package versions between /standard, /totest and /snapshot, it is important that the projects are clean prior to a checkin round. "adi" will list already staged requests, stage new requests, and supersede requests where applicable. New adi stagings will be created for new packages based on the grouping options used. The default grouping is by source project. When adi stagings are ready the request will be marked ready, unstaged, and the adi staging deleted. "check" will check if all packages are links without changes "cleanup_rings" will try to cleanup rings content and print out problems "freeze" will freeze the sources of the project's links while not affecting the source packages "frozenage" will show when the respective staging project was last frozen "ignore" will ignore a request from "list" and "adi" commands until unignored "unignore" will remove from requests from ignore list If the --cleanup flag is included then all ignored requests that were changed from state new or review more than 3 days ago will be removed. "list" will list/supersede requests for ring packages or all if no rings. The package list is used to limit what requests are superseded when called with the --supersede option. "repair" will attempt to repair the state of a request that has been corrupted. Use the --cleanup flag to include all untracked requests. "select" will add requests to the project Stagings are expected to be either in short-hand or the full project name. For example letter or named stagings can be specified simply as A, B, Gcc6, etc, while adi stagings can be specified as adi:1, adi:2, etc. Currently, adi stagings are not supported in proposal mode. Requests may either be the target package or the request ID. When using --filter-by or --group-by the xpath will be applied to the request node as returned by OBS. Several values will supplement the normal request node. - ./action/target/@devel_project: the devel project for the package - ./action/target/@ring: the ring to which the package belongs - ./@ignored: either false or the provided message Some useful examples: --filter-by './action/target[starts-with(@package, "yast-")]' --filter-by './action/target/[@devel_project="YaST:Head"]' --filter-by './action/target[starts-with(@ring, "1")]' --filter-by '@id!="1234567"' --group-by='./action/target/@devel_project' --group-by='./action/target/@ring' Multiple filter-by or group-by options may be used at the same time. Note that when using proposal mode, multiple stagings to consider may be provided in addition to a list of requests by which to filter. A more complex example: select --group-by='./action/target/@devel_project' A B C 123 456 789 This will separate the requests 123, 456, 789 by devel project and only consider stagings A, B, or C, if available, for placement. No arguments is also a valid choice and will propose all non-ignored requests into the first available staging. Note that bootstrapped stagings are only used when either required or no other stagings are available. Another useful example is placing all open requests into a specific letter staging with: select A Interactive mode allows the proposal to be modified before application. "unselect" will remove from the project - pushing them back to the backlog If a message is included the requests will be ignored first. Use the --cleanup flag to include all obsolete requests. "unlock" will remove the staging lock in case it gets stuck "rebuild" will rebuild broken packages in the given stagings or all The rebuild command will only trigger builds for packages with less than 3 failures since the last success or if the build log indicates a stall. If the force option is included the rebuild checks will be ignored and all packages failing to build will be triggered. Usage: osc staging accept [--force] [--no-cleanup] [LETTER...] osc staging acheck osc staging adi [--move] [--by-develproject] [--split] [REQUEST...] osc staging check [--old] STAGING osc staging cleanup_rings osc staging freeze [--no-boostrap] STAGING... osc staging frozenage [STAGING...] osc staging ignore [-m MESSAGE] REQUEST... osc staging unignore [--cleanup] [REQUEST...|all] osc staging list [--supersede] [PACKAGE...] osc staging select [--no-freeze] [--move [--from STAGING]] [--add PACKAGE] STAGING REQUEST... osc staging select [--no-freeze] [--interactive|--non-interactive] [--filter-by...] [--group-by...] [--merge] [--try-strategies] [--strategy] [STAGING...] [REQUEST...] osc staging unselect [--cleanup] [-m MESSAGE] [REQUEST...] osc staging unlock osc staging rebuild [--force] [STAGING...] osc staging repair [--cleanup] [REQUEST...] osc staging setprio [STAGING...] """ if opts.version: self._print_version() # verify the argument counts match the commands if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help staging"!') cmd = args[0] if cmd == 'freeze': min_args, max_args = 1, None elif cmd == 'repair': min_args, max_args = 0, None elif cmd == 'frozenage': min_args, max_args = 0, None elif cmd == 'setprio': min_args, max_args = 0, None elif cmd == 'check': min_args, max_args = 0, 1 elif cmd == 'select': min_args, max_args = 0, None elif cmd == 'unselect': min_args, max_args = 0, None elif cmd == 'adi': min_args, max_args = 0, None elif cmd == 'ignore': min_args, max_args = 1, None elif cmd == 'unignore': min_args, max_args = 0, None elif cmd in ('list', 'accept'): min_args, max_args = 0, None elif cmd in ('cleanup_rings', 'acheck'): min_args, max_args = 0, 0 elif cmd == 'unlock': min_args, max_args = 0, 0 elif cmd == 'rebuild': min_args, max_args = 0, None else: raise oscerr.WrongArgs('Unknown command: %s' % cmd) if len(args) - 1 < min_args: raise oscerr.WrongArgs('Too few arguments.') if max_args is not None and len(args) - 1 > max_args: raise oscerr.WrongArgs('Too many arguments.') # Init the OBS access and configuration opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False Config(opts.project) if opts.wipe_cache: Cache.delete_all() lock = OBSLock(opts.apiurl, opts.project) if cmd == 'unlock': lock.release() return with lock: api = StagingAPI(opts.apiurl, opts.project) # call the respective command and parse args by need if cmd == 'check': prj = args[1] if len(args) > 1 else None CheckCommand(api).perform(prj, opts.old) elif cmd == 'freeze': for prj in args[1:]: FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap=opts.bootstrap) elif cmd == 'frozenage': projects = api.get_staging_projects_short() if len( args) == 1 else args[1:] for prj in projects: print("%s last frozen %0.1f days ago" % (api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj)))) elif cmd == 'acheck': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') print( "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty)) else: print("acheck is unavailable in %s!\n" % (api.project)) elif cmd == 'accept': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest is None or opts.force: # SLE does not have a totest_version or openqa_version - ignore it version_openqa = version_totest totest_dirty = False else: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') if version_openqa == version_totest and not totest_dirty: cmd = AcceptCommand(api) for prj in args[1:]: if cmd.perform(api.prj_from_letter(prj), opts.force): cmd.reset_rebuild_data(prj) else: return if not opts.no_cleanup: if api.item_exists(api.prj_from_letter(prj)): cmd.cleanup(api.prj_from_letter(prj)) if api.item_exists("%s:DVD" % api.prj_from_letter(prj)): cmd.cleanup("%s:DVD" % api.prj_from_letter(prj)) if opts.project.startswith('openSUSE:'): cmd.accept_other_new() cmd.update_factory_version() if api.item_exists(api.crebuild): cmd.sync_buildfailures() else: print("Not safe to accept: /totest is not yet synced") elif cmd == 'unselect': if opts.message: print('Ignoring requests first') IgnoreCommand(api).perform(args[1:], opts.message) UnselectCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'select': # Include list of all stagings in short-hand and by full name. existing_stagings = api.get_staging_projects_short(None) existing_stagings += [ p for p in api.get_staging_projects() if not p.endswith(':DVD') ] stagings = [] requests = [] for arg in args[1:]: # Since requests may be given by either request ID or package # name and stagings may include multi-letter special stagings # there is no easy way to distinguish between stagings and # requests in arguments. Therefore, check if argument is in the # list of short-hand and full project name stagings, otherwise # consider it a request. This also allows for special stagings # with the same name as package, but the staging will be assumed # first time around. The current practice seems to be to start a # special staging with a capital letter which makes them unique. # lastly adi stagings are consistently prefix with adi: which # also makes it consistent to distinguish them from request IDs. if arg in existing_stagings and arg not in stagings: stagings.append(api.extract_staging_short(arg)) elif arg not in requests: requests.append(arg) if len(stagings) != 1 or len( requests) == 0 or opts.filter_by or opts.group_by: if opts.move or opts.from_: print( '--move and --from must be used with explicit staging and request list' ) return open_requests = api.get_open_requests() if len(open_requests) == 0: print('No open requests to consider') return splitter = RequestSplitter(api, open_requests, in_ring=True) considerable = splitter.stagings_load(stagings) if considerable == 0: print('No considerable stagings on which to act') return if opts.merge: splitter.merge() if opts.try_strategies: splitter.strategies_try() if len(requests) > 0: splitter.strategy_do('requests', requests=requests) if opts.strategy: splitter.strategy_do(opts.strategy) elif opts.filter_by or opts.group_by: kwargs = {} if opts.filter_by: kwargs['filters'] = opts.filter_by if opts.group_by: kwargs['groups'] = opts.group_by splitter.strategy_do('custom', **kwargs) else: if opts.merge: # Merge any none strategies before final none strategy. splitter.merge(strategy_none=True) splitter.strategy_do('none') splitter.strategy_do_non_bootstrapped('none') proposal = splitter.proposal if len(proposal) == 0: print('Empty proposal') return if opts.interactive: with tempfile.NamedTemporaryFile(suffix='.yml') as temp: temp.write( yaml.safe_dump(splitter.proposal, default_flow_style=False) + '\n\n') temp.write( '# move requests between stagings or comment/remove them\n' ) temp.write('# change the target staging for a group\n') temp.write('# stagings\n') if opts.merge: temp.write('# - merged: {}\n'.format(', '.join( sorted(splitter.stagings_mergeable + splitter.stagings_mergeable_none)))) temp.write('# - considered: {}\n'.format(', '.join( sorted(splitter.stagings_considerable)))) temp.write('# - remaining: {}\n'.format(', '.join( sorted(splitter.stagings_available)))) temp.flush() editor = os.getenv('EDITOR') if not editor: editor = 'xdg-open' return_code = subprocess.call([editor, temp.name]) proposal = yaml.safe_load(open(temp.name).read()) # Filter invalidated groups from proposal. keys = ['group', 'requests', 'staging', 'strategy'] for group, info in sorted(proposal.items()): for key in keys: if not info.get(key): del proposal[group] break print(yaml.safe_dump(proposal, default_flow_style=False)) print('Accept proposal? [y/n] (y): ', end='') if opts.non_interactive: print('y') else: response = raw_input().lower() if response != '' and response != 'y': print('Quit') return for group, info in sorted(proposal.items()): print('Staging {} in {}'.format(group, info['staging'])) # SelectCommand expects strings. request_ids = map(str, info['requests'].keys()) target_project = api.prj_from_short(info['staging']) if 'merge' not in info: # Assume that the original splitter_info is desireable # and that this staging is simply manual followup. api.set_splitter_info_in_prj_pseudometa( target_project, info['group'], info['strategy']) SelectCommand(api, target_project) \ .perform(request_ids, no_freeze=opts.no_freeze) else: target_project = api.prj_from_short(stagings[0]) if opts.add: api.mark_additional_packages(target_project, [opts.add]) else: SelectCommand(api, target_project) \ .perform(requests, opts.move, opts.from_, opts.no_freeze) elif cmd == 'cleanup_rings': CleanupRings(api).perform() elif cmd == 'ignore': IgnoreCommand(api).perform(args[1:], opts.message) elif cmd == 'unignore': UnignoreCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'list': ListCommand(api).perform(args[1:], supersede=opts.supersede) elif cmd == 'adi': AdiCommand(api).perform(args[1:], move=opts.move, by_dp=opts.by_develproject, split=opts.split) elif cmd == 'rebuild': RebuildCommand(api).perform(args[1:], opts.force) elif cmd == 'repair': RepairCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'setprio': PrioCommand(api).perform(args[1:])
def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects ${cmd_option_list} "accept" will accept all requests in openSUSE:Factory:Staging:<LETTER> (into Factory) "acheck" will check if it's safe to accept new staging projects As openSUSE:Factory is syncing the right package versions between /standard, /totest and /snapshot, it's important that the projects are clean prior to a checkin round. "check" will check if all packages are links without changes "cleanup_rings" will try to cleanup rings content and print out problems "freeze" will freeze the sources of the project's links (not affecting the packages actually in) "frozenage" will show when the respective staging project was last frozen "list" will pick the requests not in rings "select" will add requests to the project "unselect" will remove from the project - pushing them back to the backlog Usage: osc staging accept [--force] [LETTER...] osc staging check [--old] REPO osc staging cleanup_rings osc staging freeze [--no-boostrap] PROJECT... osc staging frozenage PROJECT... osc staging list [--supersede] osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST... osc staging unselect REQUEST... osc staging repair REQUEST... """ if opts.version: self._print_version() # verify the argument counts match the commands if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help staging"!') cmd = args[0] if cmd in ('freeze', 'frozenage', 'repair'): min_args, max_args = 1, None elif cmd == 'check': min_args, max_args = 0, 2 elif cmd == 'select': min_args, max_args = 1, None if not opts.add: min_args = 2 elif cmd == 'unselect': min_args, max_args = 1, None elif cmd == 'adi': min_args, max_args = None, None elif cmd in ('list', 'accept'): min_args, max_args = 0, None elif cmd in ('cleanup_rings', 'acheck'): min_args, max_args = 0, 0 else: raise oscerr.WrongArgs('Unknown command: %s' % cmd) if len(args) - 1 < min_args: raise oscerr.WrongArgs('Too few arguments.') if max_args is not None and len(args) - 1 > max_args: raise oscerr.WrongArgs('Too many arguments.') # Init the OBS access and configuration opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False Config(opts.project) with OBSLock(opts.apiurl, opts.project): api = StagingAPI(opts.apiurl, opts.project) # call the respective command and parse args by need if cmd == 'check': prj = args[1] if len(args) > 1 else None CheckCommand(api).perform(prj, opts.old) elif cmd == 'freeze': for prj in args[1:]: FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap=opts.bootstrap) elif cmd == 'frozenage': for prj in args[1:]: print "%s last frozen %0.1f days ago" % (api.prj_from_letter( prj), api.days_since_last_freeze(api.prj_from_letter(prj))) elif cmd == 'acheck': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') print "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % ( version_openqa, version_totest, totest_dirty) else: print "acheck is unavailable in %s!\n" % (api.project) elif cmd == 'accept': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest is None or opts.force: # SLE does not have a totest_version or openqa_version - ignore it version_openqa = version_totest totest_dirty = False else: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') if version_openqa == version_totest and not totest_dirty: cmd = AcceptCommand(api) for prj in args[1:]: if not cmd.perform(api.prj_from_letter(prj), opts.force): return if not opts.no_cleanup: if api.item_exists(api.prj_from_letter(prj)): cmd.cleanup(api.prj_from_letter(prj)) if api.item_exists("%s:DVD" % api.prj_from_letter(prj)): cmd.cleanup("%s:DVD" % api.prj_from_letter(prj)) if opts.project.startswith('openSUSE:'): cmd.accept_other_new() cmd.update_factory_version() if api.item_exists(api.crebuild): cmd.sync_buildfailures() else: print "Not safe to accept: /totest is not yet synced" elif cmd == 'unselect': UnselectCommand(api).perform(args[1:]) elif cmd == 'select': tprj = api.prj_from_letter(args[1]) if opts.add: api.mark_additional_packages(tprj, [opts.add]) else: SelectCommand(api, tprj).perform(args[2:], opts.move, opts.from_, opts.no_freeze) elif cmd == 'cleanup_rings': CleanupRings(api).perform() elif cmd == 'list': ListCommand(api).perform(args[1:], supersede=opts.supersede) elif cmd == 'adi': AdiCommand(api).perform(args[1:], move=opts.move, by_dp=opts.by_develproject, split=opts.split) elif cmd == 'repair': RepairCommand(api).perform(args[1:])
class ToTestBase(object): """Base class to store the basic interface""" def __init__(self, project, dryrun=False): self.project = project self.dryrun = dryrun self.api = StagingAPI(osc.conf.config['apiurl'], project='openSUSE:%s' % project) self.openqa = OpenQA_Client(server='https://openqa.opensuse.org') self.issues_to_ignore = [] self.issuefile = "{}_{}".format(self.project, ISSUE_FILE) if os.path.isfile(self.issuefile): with open(self.issuefile, 'r') as f: for line in f.readlines(): self.issues_to_ignore.append(line.strip()) def openqa_group(self): return self.project def iso_prefix(self): return self.project def jobs_num(self): return 70 def current_version(self): return self.release_version() def binaries_of_product(self, project, product): url = self.api.makeurl(['build', project, 'images', 'local', product]) try: f = self.api.retried_GET(url) except urllib2.HTTPError: return [] ret = [] root = ET.parse(f).getroot() for binary in root.findall('binary'): ret.append(binary.get('filename')) return ret def get_current_snapshot(self): """Return the current snapshot in :ToTest""" # for now we hardcode all kind of things for binary in self.binaries_of_product( 'openSUSE:%s:ToTest' % self.project, '_product:openSUSE-cd-mini-%s' % self.arch()): result = re.match( r'openSUSE-%s-NET-.*-Snapshot(.*)-Media.iso' % self.iso_prefix(), binary) if result: return result.group(1) return None def ftp_build_version(self, project, tree): for binary in self.binaries_of_product('openSUSE:%s' % project, tree): result = re.match(r'openSUSE.*Build(.*)-Media1.report', binary) if result: return result.group(1) raise Exception("can't find %s version" % self.project) def iso_build_version(self, project, tree): for binary in self.binaries_of_product('openSUSE:%s' % project, tree): result = re.match(r'openSUSE.*Build(.*)-Media.iso', binary) if result: return result.group(1) raise Exception("can't find %s version" % self.project) def release_version(self): url = self.api.makeurl([ 'build', 'openSUSE:%s' % self.project, 'standard', self.arch(), '_product:openSUSE-release' ]) f = self.api.retried_GET(url) root = ET.parse(f).getroot() for binary in root.findall('binary'): binary = binary.get('filename', '') result = re.match(r'.*-([^-]*)-[^-]*.src.rpm', binary) if result: return result.group(1) raise Exception("can't find %s version" % self.project) def find_openqa_results(self, snapshot): """Return the openqa jobs of a given snapshot and filter out the cloned jobs """ url = makeurl('https://openqa.opensuse.org', ['api', 'v1', 'jobs'], { 'group': self.openqa_group(), 'build': snapshot }) f = self.api.retried_GET(url) jobs = [] for job in json.load(f)['jobs']: if job['clone_id'] or job['result'] == 'obsoleted': continue job['name'] = job['name'].replace(snapshot, '') jobs.append(job) return jobs def _result2str(self, result): if result == QA_INPROGRESS: return 'inprogress' elif result == QA_FAILED: return 'failed' else: return 'passed' def find_failed_module(self, testmodules): # print json.dumps(testmodules, sort_keys=True, indent=4) for module in testmodules: if module['result'] != 'failed': continue flags = module['flags'] if 'fatal' in flags or 'important' in flags: return module['name'] break logger.info('%s %s %s' % (module['name'], module['result'], module['flags'])) def overall_result(self, snapshot): """Analyze the openQA jobs of a given snapshot Returns a QAResult""" if snapshot is None: return QA_FAILED jobs = self.find_openqa_results(snapshot) if len(jobs) < self.jobs_num(): # not yet scheduled logger.warning('we have only %s jobs' % len(jobs)) return QA_INPROGRESS number_of_fails = 0 in_progress = False for job in jobs: # print json.dumps(job, sort_keys=True, indent=4) if job['result'] in ('failed', 'incomplete', 'skipped', 'user_cancelled', 'obsoleted'): jobname = job['name'] # print json.dumps(job, sort_keys=True, indent=4), jobname url = makeurl( 'https://openqa.opensuse.org', ['api', 'v1', 'jobs', str(job['id']), 'comments']) f = self.api.retried_GET(url) comments = json.load(f) refs = set() #pprint(comments) labeled = 0 to_ignore = False for comment in comments: for ref in comment['bugrefs']: refs.add(str(ref)) if comment['userName'] == 'ttm' and comment[ 'text'] == 'label:unknown_failure': labeled = comment['id'] if comment['text'].find('@ttm ignore') >= 0: to_ignore = True ignored = len(refs) > 0 for ref in refs: if ref not in self.issues_to_ignore: if to_ignore: self.issues_to_ignore.append(ref) with open(self.issuefile, 'a') as f: f.write("%s\n" % ref) else: ignored = False if not ignored: number_of_fails += 1 if not labeled: data = {'text': 'label:unknown_failure'} self.openqa.openqa_request('POST', 'jobs/%s/comments' % job['id'], data=data) elif labeled: # remove flag - unfortunately can't delete comment unless admin data = {'text': 'Ignored issue'} self.openqa.openqa_request('PUT', 'jobs/%s/comments/%d' % (job['id'], labeled), data=data) if ignored: logger.info("job %s failed, but was ignored", jobname) else: joburl = 'https://openqa.opensuse.org/tests/%s' % job['id'] logger.info("job %s failed, see %s", jobname, joburl) elif job['result'] == 'passed' or job['result'] == 'softfailed': continue elif job['result'] == 'none': if job['state'] != 'cancelled': in_progress = True else: raise Exception(job['result']) if number_of_fails > 0: return QA_FAILED if in_progress: return QA_INPROGRESS return QA_PASSED def all_repos_done(self, project, codes=None): """Check the build result of the project and only return True if all repos of that project are either published or unpublished """ # coolo's experience says that 'finished' won't be # sufficient here, so don't try to add it :-) codes = ['published', 'unpublished'] if not codes else codes url = self.api.makeurl(['build', project, '_result'], {'code': 'failed'}) f = self.api.retried_GET(url) root = ET.parse(f).getroot() ready = True for repo in root.findall('result'): # ignore ports. 'factory' is used by arm for repos that are not # meant to use the totest manager. if repo.get('repository') in ('ports', 'factory', 'images_staging'): continue # ignore 32bit for now. We're only interesed in aarch64 here if repo.get('arch') in ('armv6l', 'armv7l'): continue if repo.get('dirty', '') == 'true': logger.info('%s %s %s -> %s' % (repo.get('project'), repo.get('repository'), repo.get('arch'), 'dirty')) ready = False if repo.get('code') not in codes: logger.info('%s %s %s -> %s' % (repo.get('project'), repo.get('repository'), repo.get('arch'), repo.get('code'))) ready = False return ready def maxsize_for_package(self, package): if re.match(r'.*-mini-.*', package): return 737280000 # a CD needs to match if re.match(r'.*-dvd5-.*', package): return 4700372992 # a DVD needs to match if re.match(r'.*-image-livecd-x11.*', package): return 681574400 # not a full CD if re.match(r'.*-image-livecd.*', package): return 999999999 # a GB stick if re.match(r'.*-dvd9-dvd-.*', package): return 8539996159 if package.startswith('_product:openSUSE-ftp-ftp-'): return None if package.startswith('_product:openSUSE-Addon-NonOss-ftp-ftp'): return None raise Exception('No maxsize for {}'.format(package)) def package_ok(self, project, package, repository, arch): """Checks one package in a project and returns True if it's succeeded """ query = {'package': package, 'repository': repository, 'arch': arch} url = self.api.makeurl(['build', project, '_result'], query) f = self.api.retried_GET(url) root = ET.parse(f).getroot() for repo in root.findall('result'): status = repo.find('status') if status.get('code') != 'succeeded': logger.info( '%s %s %s %s -> %s' % (project, package, repository, arch, status.get('code'))) return False maxsize = self.maxsize_for_package(package) if not maxsize: return True url = self.api.makeurl(['build', project, repository, arch, package]) f = self.api.retried_GET(url) root = ET.parse(f).getroot() for binary in root.findall('binary'): if not binary.get('filename', '').endswith('.iso'): continue isosize = int(binary.get('size', 0)) if isosize > maxsize: logger.error('%s %s %s %s: %s' % (project, package, repository, arch, 'too large by %s bytes' % (isosize - maxsize))) return False return True def is_snapshottable(self): """Check various conditions required for factory to be snapshotable """ if not self.all_repos_done('openSUSE:%s' % self.project): return False for product in self.ftp_products + self.main_products: if not self.package_ok('openSUSE:%s' % self.project, product, 'images', 'local'): return False if len(self.livecd_products): if not self.all_repos_done('openSUSE:%s:Live' % self.project): return False for arch in ['i586', 'x86_64']: for product in self.livecd_products: if not self.package_ok( 'openSUSE:%s:Live' % self.project, product, 'standard', arch): return False return True def release_package(self, project, package, set_release=None): query = {'cmd': 'release'} if set_release: query['setrelease'] = set_release # FIXME: make configurable. openSUSE:Factory:ARM currently has multiple # repos with release targets, so obs needs to know which one to release if project == 'openSUSE:Factory:ARM': query['repository'] = 'images' baseurl = ['source', project, package] url = self.api.makeurl(baseurl, query=query) if self.dryrun: logger.info("release %s/%s (%s)" % (project, package, set_release)) else: self.api.retried_POST(url) def update_totest(self, snapshot=None): release = 'Snapshot%s' % snapshot if snapshot else None logger.info('Updating snapshot %s' % snapshot) if not self.dryrun: self.api.switch_flag_in_prj('openSUSE:%s:ToTest' % self.project, flag='publish', state='disable') for product in self.ftp_products: self.release_package('openSUSE:%s' % self.project, product) for cd in self.livecd_products: self.release_package('openSUSE:%s:Live' % self.project, cd, set_release=release) for cd in self.main_products: self.release_package('openSUSE:%s' % self.project, cd, set_release=release) def publish_factory_totest(self): logger.info('Publish ToTest') if not self.dryrun: self.api.switch_flag_in_prj('openSUSE:%s:ToTest' % self.project, flag='publish', state='enable') def totest_is_publishing(self): """Find out if the publishing flag is set in totest's _meta""" url = self.api.makeurl( ['source', 'openSUSE:%s:ToTest' % self.project, '_meta']) f = self.api.retried_GET(url) root = ET.parse(f).getroot() if not root.find('publish'): # default true return True for flag in root.find('publish'): if flag.get('repository', None) or flag.get('arch', None): continue if flag.tag == 'enable': return True return False def totest(self): current_snapshot = self.get_current_snapshot() new_snapshot = self.current_version() current_result = self.overall_result(current_snapshot) current_qa_version = self.api.load_file_content( "%s:Staging" % self.api.project, "dashboard", "version_totest") logger.info('current_snapshot %s: %s' % (current_snapshot, self._result2str(current_result))) logger.debug('new_snapshot %s', new_snapshot) logger.debug('current_qa_version %s', current_qa_version) snapshotable = self.is_snapshottable() logger.debug("snapshotable: %s", snapshotable) can_release = (current_result != QA_INPROGRESS and snapshotable) # not overwriting if new_snapshot == current_snapshot: logger.debug("no change in snapshot version") can_release = False elif not self.all_repos_done('openSUSE:%s:ToTest' % self.project): logger.debug("not all repos done, can't release") # the repos have to be done, otherwise we better not touch them # with a new release can_release = False can_publish = (current_result == QA_PASSED) # already published if self.totest_is_publishing(): logger.debug("totest already publishing") can_publish = False if can_publish: if current_qa_version == current_snapshot: self.publish_factory_totest() self.write_version_to_dashboard("snapshot", current_snapshot) can_release = False # we have to wait else: # We reached a very bad status: openQA testing is 'done', but not of the same version # currently in :ToTest. This can happen when 'releasing' the # product failed raise Exception( "Publishing stopped: tested version (%s) does not match :ToTest version (%s)" % (current_qa_version, current_snapshot)) if can_release: self.update_totest(new_snapshot) self.write_version_to_dashboard("totest", new_snapshot) def release(self): new_snapshot = self.current_version() self.update_totest(new_snapshot) def write_version_to_dashboard(self, target, version): if not self.dryrun: url = self.api.makeurl([ 'source', 'openSUSE:%s:Staging' % self.project, 'dashboard', 'version_%s' % target ]) osc.core.http_PUT(url + '?comment=Update+version', data=version)
def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects ${cmd_option_list} "accept" will accept all requests in openSUSE:Factory:Staging:<LETTER> (into Factory) "acheck" will check if it's safe to accept new staging projects As openSUSE:Factory is syncing the right package versions between /standard, /totest and /snapshot, it's important that the projects are clean prior to a checkin round. "check" will check if all packages are links without changes "cleanup_rings" will try to cleanup rings content and print out problems "freeze" will freeze the sources of the project's links (not affecting the packages actually in) "frozenage" will show when the respective staging project was last frozen "list" will pick the requests not in rings "select" will add requests to the project "unselect" will remove from the project - pushing them back to the backlog Usage: osc staging accept [--force] [LETTER...] osc staging check [--old] REPO osc staging cleanup_rings osc staging freeze [--no-boostrap] PROJECT... osc staging frozenage PROJECT... osc staging list [--supersede] osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST... osc staging unselect REQUEST... osc staging repair REQUEST... """ if opts.version: self._print_version() # verify the argument counts match the commands if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help staging"!') cmd = args[0] if cmd in ("freeze", "frozenage", "repair"): min_args, max_args = 1, None elif cmd == "check": min_args, max_args = 0, 2 elif cmd == "select": min_args, max_args = 1, None if not opts.add: min_args = 2 elif cmd == "unselect": min_args, max_args = 1, None elif cmd == "adi": min_args, max_args = None, None elif cmd in ("list", "accept"): min_args, max_args = 0, None elif cmd in ("cleanup_rings", "acheck"): min_args, max_args = 0, 0 else: raise oscerr.WrongArgs("Unknown command: %s" % cmd) if len(args) - 1 < min_args: raise oscerr.WrongArgs("Too few arguments.") if max_args is not None and len(args) - 1 > max_args: raise oscerr.WrongArgs("Too many arguments.") # Init the OBS access and configuration opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False Config(opts.project) with OBSLock(opts.apiurl, opts.project): api = StagingAPI(opts.apiurl, opts.project) # call the respective command and parse args by need if cmd == "check": prj = args[1] if len(args) > 1 else None CheckCommand(api).perform(prj, opts.old) elif cmd == "freeze": for prj in args[1:]: FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap=opts.bootstrap) elif cmd == "frozenage": for prj in args[1:]: print "%s last frozen %0.1f days ago" % ( api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj)), ) elif cmd == "acheck": # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version( api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64" ) if version_totest: version_openqa = api.load_file_content("%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, "totest") print "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % ( version_openqa, version_totest, totest_dirty, ) else: print "acheck is unavailable in %s!\n" % (api.project) elif cmd == "accept": # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version( api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64" ) if version_totest is None or opts.force: # SLE does not have a totest_version or openqa_version - ignore it version_openqa = version_totest totest_dirty = False else: version_openqa = api.load_file_content("%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, "totest") if version_openqa == version_totest and not totest_dirty: cmd = AcceptCommand(api) for prj in args[1:]: if not cmd.perform(api.prj_from_letter(prj), opts.force): return if not opts.no_cleanup: if api.item_exists(api.prj_from_letter(prj)): cmd.cleanup(api.prj_from_letter(prj)) if api.item_exists("%s:DVD" % api.prj_from_letter(prj)): cmd.cleanup("%s:DVD" % api.prj_from_letter(prj)) if opts.project.startswith("openSUSE:"): cmd.accept_other_new() cmd.update_factory_version() if api.item_exists(api.crebuild): cmd.sync_buildfailures() else: print "Not safe to accept: /totest is not yet synced" elif cmd == "unselect": UnselectCommand(api).perform(args[1:]) elif cmd == "select": tprj = api.prj_from_letter(args[1]) if opts.add: api.mark_additional_packages(tprj, [opts.add]) else: SelectCommand(api, tprj).perform(args[2:], opts.move, opts.from_, opts.no_freeze) elif cmd == "cleanup_rings": CleanupRings(api).perform() elif cmd == "list": ListCommand(api).perform(args[1:], supersede=opts.supersede) elif cmd == "adi": AdiCommand(api).perform(args[1:], move=opts.move, by_dp=opts.by_develproject, split=opts.split) elif cmd == "repair": RepairCommand(api).perform(args[1:])