def add_review(self, request_id, by_project=None, by_group=None, msg=None): """ Adds review by project to the request :param request_id: request to add review to :param project: project to assign review to """ req = get_request(self.apiurl, str(request_id)) if not req: raise oscerr.WrongArgs('Request {} not found'.format(request_id)) for i in req.reviews: if by_project and i.by_project == by_project and i.state == 'new': return if by_group and i.by_group == by_group and i.state == 'new': return # don't try to change reviews if the request is dead if req.state.name not in ('new', 'review'): return query = {} if by_project: query['by_project'] = by_project if not msg: msg = 'Being evaluated by staging project "{}"' msg = msg.format(by_project) if by_group: query['by_group'] = by_group if not msg: msg = 'Being evaluated by group "{}"'.format(by_group) if not query: raise oscerr.WrongArgs('We need a group or a project') query['cmd'] = 'addreview' url = self.makeurl(['request', str(request_id)], query) http_POST(url, data=msg)
def repair(self, request): reviews = [] reqid = str(request) req = get_request(self.api.apiurl, reqid) if not req: raise oscerr.WrongArgs('Request {} not found'.format(reqid)) if req.state.name != 'review': print('Request "{}" is not in review state'.format(reqid)) return reviews = [ r.by_project for r in req.reviews if ':Staging:' in str(r.by_project) and r.state == 'new' ] if reviews: if len(reviews) > 1: raise oscerr.WrongArgs( 'Request {} had multiple review opened by different staging project' .format(reqid)) else: raise oscerr.WrongArgs( 'Request {} is not for staging project'.format(reqid)) staging_project = reviews[0] try: data = self.api.get_prj_pseudometa(staging_project) except urllib2.HTTPError, e: if e.code == 404: data = None
def do_vdelreq(self, subcmd, opts, *args): """${cmd_name}: display pending virtual accept delete request osc vdelreq [OPT] COMMAND PROJECT Shows pending the virtual accept delete requests and the current state. ${cmd_option_list} "list" will list virtually accepted delete request. Usage: osc vdelreq [--delreq-review DELREQREVIEW] list PROJECT """ self.apiurl = self.get_api_url() if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help vdelreq"!') if len(args) < 2: raise oscerr.WrongArgs('No project given, see "osc help vdelreq"!') cmd = args[0] if cmd in ('list'): self.list_virtually_accepted_request(args[1], opts) else: raise oscerr.WrongArgs('Unknown command: %s' % cmd)
def do_origin(self, subcmd, opts, *args): """${cmd_name}: tools for working with origin information ${cmd_option_list} config: print expanded OSRT:OriginConfig cron: update the lookup for all projects with an OSRT:OriginConfig attribute history: list requests containing an origin annotation list: print all packages and their origin package: print the origin of package potentials: list potential origins of a package projects: list all projects with an OSRT:OriginConfig attribute report: print origin summary report update: handle package source changes as either delete or submit requests Usage: osc origin config [--origins-only] osc origin cron osc origin history [--format json|yaml] PACKAGE osc origin list [--force-refresh] [--format json|yaml] osc origin package [--debug] PACKAGE osc origin potentials [--format json|yaml] PACKAGE osc origin projects [--format json|yaml] osc origin report [--diff] [--force-refresh] [--mail] osc origin update [--listen] [--listen-seconds] [PACKAGE...] """ if len(args) == 0: raise oscerr.WrongArgs('A command must be indicated.') command = args[0] if command not in [ 'config', 'cron', 'history', 'list', 'package', 'potentials', 'projects', 'report', 'update' ]: raise oscerr.WrongArgs('Unknown command: {}'.format(command)) if command == 'package' and len(args) < 2: raise oscerr.WrongArgs('A package must be indicated.') level = logging.DEBUG if opts.debug else None logging.basicConfig(level=level, format='[%(levelname).1s] %(message)s') # Allow for determining project from osc store. if not opts.project and core.is_project_dir('.'): opts.project = core.store_read_project('.') Cache.init() apiurl = self.get_api_url() if command not in ['cron', 'projects', 'update']: if not opts.project: raise oscerr.WrongArgs('A project must be indicated.') config = config_load(apiurl, opts.project) if not config: raise oscerr.WrongArgs( 'OSRT:OriginConfig attribute missing from {}'.format( opts.project)) function = 'osrt_origin_{}'.format(command) globals()[function](apiurl, opts, *args[1:])
def repair(self, request): reviews = [] reqid = str(request) req = get_request(self.api.apiurl, reqid) if not req: raise oscerr.WrongArgs('Request {} not found'.format(reqid)) if req.state.name != 'review': print('Request "{}" is not in review state'.format(reqid)) return reviews = [ r.by_project for r in req.reviews if ':Staging:' in str(r.by_project) and r.state == 'new' ] if reviews: if len(reviews) > 1: raise oscerr.WrongArgs( 'Request {} had multiple review opened by different staging project' .format(reqid)) else: raise oscerr.WrongArgs( 'Request {} is not for staging project'.format(reqid)) staging_project = reviews[0] try: data = self.api.project_status(staging_project) except HTTPError as e: if e.code == 404: data = None # Pre-check and pre-setup if data is not None: for request in data.findall('staged_requests/requests'): if request.get('id') == reqid: print('Request "{}" had the good setup in "{}"'.format( reqid, staging_project)) return else: # this situation should only happen on adi staging print('Project is not exist, re-creating "{}"'.format( staging_project)) name = self.api.create_adi_project(staging_project) # a bad request setup found print('Repairing "{}"'.format(reqid)) change_review_state(self.api.apiurl, reqid, newstate='accepted', message='Re-evaluation needed', by_project=staging_project) self.api.add_review(reqid, by_group=self.api.cstaging_group, msg='Requesting new staging review')
def find_via_stagingapi(self, pkgs): """ Search for all various mutations and return list of SR#s. Use and instance of StagingAPI to direct the search, this makes sure that the SR# are inside a staging project. :param pkgs: mesh of argumets to search for This function is only called for its side effect. """ for p in pkgs: found = False for staging in self.api.get_staging_projects(): if _is_int(p) and self.api.get_package_for_request_id( staging, p): self.srs[int(p)] = {'staging': staging} found = True continue else: rq = self.api.get_request_id_for_package(staging, p) if rq: self.srs[rq] = {'staging': staging} found = True continue if not found: raise oscerr.WrongArgs('No SR# found for: {}'.format(p))
def find_request_id(self, request_id): """ Look up the request by ID to verify if it is correct :param request_id: ID of the added request """ if not _is_int(request_id): return False url = makeurl(self.api.apiurl, ['request', str(request_id)]) try: f = http_GET(url) except urllib2.HTTPError: return None root = ET.parse(f).getroot() if root.get('id', None) != str(request_id): return None project = root.find('action').find('target').get('project') if (project != self.api.project and not project.startswith(self.api.cstaging)): msg = 'Request {} is not for {}, but for {}' msg = msg.format(request_id, self.api.project, project) raise oscerr.WrongArgs(msg) self.srs[int(request_id)] = {'project': project} return True
def supseded_request(self, request): """ Returns a staging info for a request or None :param request - a Request instance :return dict with 'prj' and 'rq_id' of the old request """ # Consolidate all data from request request_id = int(request.get('id')) action = request.findall('action') if not action: msg = 'Request {} has no action'.format(request_id) raise oscerr.WrongArgs(msg) # we care only about first action action = action[0] # Where are we targeting the package target_project = action.find('target').get('project') target_package = action.find('target').get('package') # If the values are empty it is no error if not target_project or not target_package: msg = 'no target/package in request {}, action {}; ' msg = msg.format(request_id, action) logging.info(msg) # If the package is currently tracked then we do the replacement stage_info = self.packages_staged.get(target_package, {'prj': '', 'rq_id': 0}) if stage_info['rq_id'] != 0 and int(stage_info['rq_id']) != request_id: return stage_info return None
def do_change_review_state(self, request_id, newstate, message=None, by_group=None, by_user=None, by_project=None): """ Change review state of the staging request :param request_id: id of the request :param newstate: state of the new request :param message: message for the review :param by_group, by_user, by_project: review type """ message = '' if not message else message req = get_request(self.apiurl, str(request_id)) if not req: raise oscerr.WrongArgs('Request {} not found'.format(request_id)) for review in req.reviews: if review.by_group == by_group and \ review.by_user == by_user and \ review.by_project == by_project and \ review.state == 'new': # call osc's function return change_review_state(self.apiurl, str(request_id), newstate, message=message, by_group=by_group, by_user=by_user, by_project=by_project) return False
def accept_non_ring_request(self, request): """ Accept review of requests that are not yet in any ring so we don't delay their testing. :param request: request to check """ # Consolidate all data from request request_id = int(request.get('id')) action = request.findall('action') if not action: msg = 'Request {} has no action'.format(request_id) raise oscerr.WrongArgs(msg) # we care only about first action action = action[0] # Where are we targeting the package target_project = action.find('target').get('project') target_package = action.find('target').get('package') # If the values are empty it is no error if not target_project or not target_package: msg = 'no target/package in request {}, action {}; ' msg = msg.format(request_id, action) logging.info(msg) # Verify the package ring ring = self.ring_packages.get(target_package, None) if not ring: # accept the request here message = 'No need for staging, not in tested ring projects.' self.do_change_review_state(request_id, 'accepted', message=message, by_group=self.cstaging_group)
def set_review(self, request_id, project, state='accepted', msg=None): """ Sets review for request done by project :param request_id: request to change review for :param project: project to do the review """ req = get_request(self.apiurl, str(request_id)) if not req: raise oscerr.WrongArgs('Request {} not found'.format(request_id)) # don't try to change reviews if the request is dead if req.state.name not in ('new', 'review'): return cont = False for i in req.reviews: if i.by_project == project and i.state == 'new': cont = True if not cont: return if not msg: msg = 'Reviewed by staging project "{}" with result: "{}"' msg = msg.format(project, state) self.do_change_review_state(request_id, state, by_project=project, message=msg)
def find_via_stagingapi(self, pkgs): """ Search for all various mutations and return list of SR#s. Use and instance of StagingAPI to direct the search, this makes sure that the SR# are inside a staging project. :param pkgs: mesh of argumets to search for This function is only called for its side effect. """ url = self.api.makeurl( ['staging', self.api.project, 'staging_projects'], {'requests': 1}) status = ET.parse(self.api.retried_GET(url)).getroot() for p in pkgs: found = False for staging in status.findall('staging_project'): for request in staging.findall('staged_requests/request'): if request.get('package') == p or request.get('id') == p: self.srs[int(request.get('id'))] = { 'staging': staging.get('name') } found = True break if not found: raise oscerr.WrongArgs('No SR# found for: {}'.format(p))
def find_request_package(self, package): """ Look up the package by its name and return the SR# :param package: name of the package """ query = 'types=submit,delete&states=new,review&project={}&view=collection&package={}' query = query.format(self.api.project, quote(package)) url = makeurl(self.api.apiurl, ['request'], query) f = http_GET(url) root = ET.parse(f).getroot() requests = [] for sr in root.findall('request'): # Check the target matches - OBS query is case insensitive, but OBS is not rq_target = sr.find('action').find('target') if package != rq_target.get('package') or self.api.project != rq_target.get('project'): continue request = sr.get('id') state = sr.find('state').get('name') self.srs[int(request)] = {'project': self.api.project, 'state': state} requests.append(request) if len(requests) > 1: msg = 'There are multiple requests for package "{}": {}' msg = msg.format(package, ', '.join(requests)) raise oscerr.WrongArgs(msg) request = int(requests[0]) if requests else None return request
def __init__(self, source_package=None, target_project=None, target_package=None, type='submit'): """Creates a request in the OBS instance and instantiates an object to represent it""" self.revoked = True if type == 'submit': self.reqid = osc.core.create_submit_request( APIURL, src_project=source_package.project.name, src_package=source_package.name, dst_project=target_project, dst_package=target_package) print('created submit request {}/{} -> {}'.format( source_package.project.name, source_package.name, target_project)) elif type == 'delete': self.reqid = create_delete_request(APIURL, target_project.name, target_package.name) else: raise oscerr.WrongArgs(f'unknown request type {type}') self.revoked = False
def select_request(self, request, move, filter_from): supersede = False staged_requests = { int(self.api.packages_staged[package]['rq_id']): package for package in self.api.packages_staged } if request in staged_requests: supersede = self._supersede(request) if request not in staged_requests and not supersede: # Normal 'select' command print('Adding request "{}" to project "{}"'.format( request, self.target_project)) return self.api.rq_to_prj(request, self.target_project) elif request in staged_requests and (move or supersede): # 'select' command becomes a 'move' # supersede = (new_rq, package, project) fprj = self.api.packages_staged[staged_requests[request]][ 'prj'] if not supersede else supersede[2] if filter_from and filter_from != fprj: print('Ignoring "{}" in "{}" since not in "{}"'.format( request, fprj, filter_from)) return True if supersede: print('"{} ({}) is superseded by {}'.format( request, supersede[1], supersede[0])) if fprj == self.target_project: print('"{}" is currently in "{}"'.format( request, self.target_project)) return False print('Moving "{}" from "{}" to "{}"'.format( request, fprj, self.target_project)) # Store the source project, we also need to write a comment there self.affected_projects.add(fprj) return self.api.move_between_project(fprj, request, self.target_project) elif request in staged_requests and not move: # Previously selected, but not explicit move fprj = self.api.packages_staged[staged_requests[request]]['prj'] msg = 'Request {} is already tracked in "{}".' msg = msg.format(request, fprj) if fprj != self.target_project: msg += '\nUse --move modifier to move the request from "{}" to "{}"' msg = msg.format(fprj, self.target_project) print(msg) return True elif supersede: print('"{} ({}) supersedes {}'.format(request, supersede[1], supersede[0])) else: raise oscerr.WrongArgs('Arguments for select are not correct.')
def repair(self, request): reviews = [] reqid = str(request) req = get_request(self.api.apiurl, reqid) if not req: raise oscerr.WrongArgs('Request {} not found'.format(reqid)) if req.state.name != 'review': print('Request "{}" is not in review state'.format(reqid)) return reviews = [ r.by_project for r in req.reviews if ':Staging:' in str(r.by_project) and r.state == 'new' ] if reviews: if len(reviews) > 1: raise oscerr.WrongArgs( 'Request {} had multiple review opened by different staging project' .format(reqid)) else: raise oscerr.WrongArgs( 'Request {} is not for staging project'.format(reqid)) staging_project = reviews[0] data = self.api.get_prj_pseudometa(staging_project) for request in data['requests']: if request['id'] == reqid: print('Request "{}" had the good setup in "{}"'.format( reqid, staging_project)) return # a bad request setup found print('Repairing "{}"'.format(reqid)) change_review_state(self.api.apiurl, reqid, newstate='accepted', message='Re-evaluation needed', by_project=staging_project) self.api.add_review(reqid, by_group=self.api.cstaging_group, msg='Requesting new staging review')
def do_check_repo(self, subcmd, opts, *args): """${cmd_name}: Checker review of submit requests. Usage: ${cmd_name} [SRID]... Shows pending review requests and their current state. ${cmd_option_list} """ opts.mode = '' opts.groups = {} opts.grouped = {} opts.verbose = False opts.apiurl = self.get_api_url() if opts.skip: if not len(args): raise oscerr.WrongArgs( 'Please give, if you want to skip a review specify a SRID') for id_ in args: self._check_repo_change_review_state(opts, id_, 'accepted', message='skip review') return ids = [arg for arg in args if arg.isdigit()] packs = [] if not ids: # xpath query, using the -m, -r, -s options where = "@by_user='******'+and+@state='new'" url = makeurl(opts.apiurl, ['search', 'request'], "match=state/@name='review'+and+review[" + where + "]") f = http_GET(url) root = ET.parse(f).getroot() for rq in root.findall('request'): packs.extend(self._check_repo_one_request(rq, opts)) else: # we have a list, use them. for id_ in ids: packs.extend(self._check_repo_fetch_request(id_, opts)) groups = {} for p in packs: a = groups.get(p.group, []) a.append(p) groups[p.group] = a for id_, reqs in groups.items(): self._check_repo_group(id_, reqs, opts)
def create_new_adi(self, wanted_requests): all_requests = self.api.get_open_requests() non_ring_packages = [] non_ring_requests = dict() for request in all_requests: # Consolidate all data from request request_id = int(request.get('id')) if len(wanted_requests) and request_id not in wanted_requests: continue action = request.findall('action') if not action: msg = 'Request {} has no action'.format(request_id) raise oscerr.WrongArgs(msg) # we care only about first action action = action[0] # Where are we targeting the package if len(wanted_requests): source_project = 'wanted' else: source = action.find('source') if source is not None: source_project = source.get('project') else: source_project = 'none' target_package = action.find('target').get('package') if not self.api.ring_packages.get(target_package): non_ring_packages.append(target_package) if not source_project in non_ring_requests: non_ring_requests[source_project] = [] non_ring_requests[source_project].append(request_id) if len(non_ring_packages): print "Not in a ring:", ' '.join(sorted(non_ring_packages)) else: return for source_project, requests in non_ring_requests.items(): name = self.api.create_adi_project(None) for request in requests: if not self.api.rq_to_prj(request, name): return False # Notify everybody about the changes self.api.update_status_comments(name, 'select')
def rq_to_prj(self, request_id, project): """ Links request to project - delete or submit :param request_id: request to link :param project: project to link into """ # read info from sr tar_pkg = None req = get_request(self.apiurl, str(request_id)) if not req: raise oscerr.WrongArgs('Request {} not found'.format(request_id)) act = req.get_actions('submit') if act: tar_pkg = self.submit_to_prj(act[0], project) act = req.get_actions('delete') if act: tar_pkg = self.delete_to_prj(act[0], project) if not tar_pkg: msg = 'Request {} is not a submit or delete request' msg = msg.format(request_id) raise oscerr.WrongArgs(msg) # register the package name self._add_rq_to_prj_pseudometa(project, int(request_id), tar_pkg) # add review self.add_review(request_id, project) # now remove the staging checker self.do_change_review_state(request_id, 'accepted', by_group=self.cstaging_group, message='Picked {}'.format(project)) return True
def add_review(self, requestid, by_project=None, by_package=None, msg=None): query = {} query['by_project'] = by_project query['by_package'] = by_package if not msg: msg = "Being evaluated by {}/{}. This is submitted by a tool to Leap, please review this change and decline it if Leap do not need this!" msg = msg.format(by_project, by_package) if not query: raise oscerr.WrongArgs('We need a project') query['cmd'] = 'addreview' url = makeurl(self.apiurl, ['request', str(requestid)], query) http_POST(url, data=msg)
def check_adi_project(self, project): query_project = self.api.extract_staging_short(project) info = self.api.project_status(project, reload=True) if info.find('staged_requests/request') is not None: if info.find('building_repositories/repo') is not None: print(query_project + ' ' + Fore.MAGENTA + 'building') return if info.find('untracked_requests/request') is not None: print(query_project + ' ' + Fore.YELLOW + 'untracked: ' + ', '.join(['{}[{}]'.format( Fore.CYAN + req.get('package') + Fore.RESET, req.get('id')) for req in info.findall('untracked_requests/request')])) return if info.find('obsolete_requests/request') is not None: print(query_project + ' ' + Fore.YELLOW + 'obsolete: ' + ', '.join(['{}[{}]'.format( Fore.CYAN + req.get('package') + Fore.RESET, req.get('id')) for req in info.findall('obsolete_requests/request')])) return if info.find('broken_packages/package') is not None: print(query_project + ' ' + Fore.RED + 'broken: ' + ', '.join([ Fore.CYAN + p.get('package') + Fore.RESET for p in info.findall('broken_packages/package')])) return for review in info.findall('missing_reviews/review'): print(query_project + ' ' + Fore.WHITE + 'review: ' + '{} for {}[{}]'.format( Fore.YELLOW + self.api.format_review(review) + Fore.RESET, Fore.CYAN + review.get('package') + Fore.RESET, review.get('request'))) return for check in info.findall('missing_checks/check'): print(query_project + ' ' + Fore.MAGENTA + 'missing: {}'.format(check.get('name'))) return for check in info.findall('checks/check'): state = check.find('state').text if state != 'success': print(query_project + '{} {} check: {}'.format(Fore.MAGENTA, state, check.get('name'))) return overall_state = info.get('state') if overall_state == 'empty': self.api.delete_empty_adi_project(project) return if overall_state != 'acceptable': raise oscerr.WrongArgs('Missed some case') ready = [] for req in info.findall('staged_requests/request'): ready.append('{}[{}]'.format(Fore.CYAN + req.get('package') + Fore.RESET, req.get('id'))) if len(ready): print(query_project, Fore.GREEN + 'ready:', ', '.join(ready))
def find(self, pkgs): """ Search for all various mutations and return list of SR#s :param pkgs: mesh of argumets to search for This function is only called for its side effect. """ for p in pkgs: if self.find_request_package(p): continue if self.find_request_id(p): continue if self.find_request_project(p): continue raise oscerr.WrongArgs('No SR# found for: {}'.format(p))
def find(self, pkgs, newcand): """ Search for all various mutations and return list of SR#s :param pkgs: mesh of argumets to search for :param newcand: the review state of staging-group must be new This function is only called for its side effect. """ for p in pkgs: if _is_int(p) and self.find_request_id(p): continue if self.find_request_package(p): continue if self.find_request_project(p, newcand): continue raise oscerr.WrongArgs('No SR# found for: {}'.format(p))
def find_request_id(self, request_id): """ Look up the request by ID to verify if it is correct :param request_id: ID of the added request """ root = self.load_request(request_id) if root is None: return False project = root.find('action').find('target').get('project') if (project != self.api.project and not project.startswith(self.api.cstaging)): msg = 'Request {} is not for {}, but for {}' msg = msg.format(request_id, self.api.project, project) raise oscerr.WrongArgs(msg) self.srs[int(request_id)] = {'project': project} return True
def perform(self): """ Perform the list command """ # First dispatch all possible requests self.api.dispatch_open_requests() # Print out the left overs requests = self.api.get_open_requests() non_ring_packages = [] for request in requests: # Consolidate all data from request request_id = int(request.get('id')) action = request.findall('action') if not action: msg = 'Request {} has no action'.format(request_id) raise oscerr.WrongArgs(msg) # we care only about first action action = action[0] # Where are we targeting the package target_package = action.find('target').get('package') # If the system have rings, we ask for the ring of the # package if self.api.crings: ring = self.api.ring_packages.get(target_package) else: ring = self.api.project # This condition is quite moot as we dispatched stuff # above anyway if ring: print('Request({}): {} -> {}'.format(request_id, target_package, ring)) else: non_ring_packages.append(target_package) if len(non_ring_packages): print "Not in a ring:", ' '.join(sorted(non_ring_packages))
def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects ${cmd_option_list} "accept" will accept all requests in $PROJECT:Staging:<LETTER> into $PROJECT If openSUSE:* project, requests marked ready from adi stagings will also be accepted. "acheck" will check if it is safe to accept new staging projects As $PROJECT is syncing the right package versions between /standard, /totest and /snapshot, it is important that the projects are clean prior to a checkin round. "adi" will list already staged requests, stage new requests, and supersede requests where applicable. New adi stagings will be created for new packages based on the grouping options used. The default grouping is by source project. When adi stagings are ready the request will be marked ready, unstaged, and the adi staging deleted. "check" will check if all packages are links without changes "check_duplicate_binaries" list binaries provided by multiple packages "config" will modify or view staging specific configuration Target project OSRT:Config attribute configuration applies to all stagings. Both configuration locations follow the .oscrc format (space separated list). config Print all staging configuration. config key Print the value of key for stagings. conf key value... Set the value of key for stagings. config --clear Clear all staging configuration. config --clear key Clear (unset) a single key from staging configuration config --append key value... Append value to existing value or set if no existing value. All of the above may be restricted to a set of stagings. The staging configuration is automatically cleared anytime staging psuedometa is cleared (accept, or unstage all requests). The keys that may be set in staging configuration are: - repo_checker-binary-whitelist[-arch]: appended to target project list - todo: text to be printed after staging is accepted "cleanup_rings" will try to cleanup rings content and print out problems "freeze" will freeze the sources of the project's links while not affecting the source packages "frozenage" will show when the respective staging project was last frozen "ignore" will ignore a request from "list" and "adi" commands until unignored "unignore" will remove from requests from ignore list If the --cleanup flag is included then all ignored requests that were changed from state new or review more than 3 days ago will be removed. "list" will list/supersede requests for ring packages or all if no rings. "lock" acquire a hold on the project in order to execute multiple commands and prevent others from interrupting. An example: lock -m "checkin round" list --supersede adi accept A B C D E unlock Each command will update the lock to keep it up-to-date. "repair" will attempt to repair the state of a request that has been corrupted. Use the --cleanup flag to include all untracked requests. "select" will add requests to the project Stagings are expected to be either in short-hand or the full project name. For example letter or named stagings can be specified simply as A, B, Gcc6, etc, while adi stagings can be specified as adi:1, adi:2, etc. Currently, adi stagings are not supported in proposal mode. Requests may either be the target package or the request ID. When using --filter-by or --group-by the xpath will be applied to the request node as returned by OBS. Use the following on a current request to see the XML structure. osc api /request/1337 A number of additional values will supplement the normal request node. - ./action/target/@devel_project: the devel project for the package - ./action/target/@devel_project_super: super devel project if relevant - ./action/target/@ring: the ring to which the package belongs - ./@aged: either True or False based on splitter-request-age-threshold - ./@nonfree: set to nonfree if targetting nonfree sub project - ./@ignored: either False or the provided message Some useful examples: --filter-by './action/target[starts-with(@package, "yast-")]' --filter-by './action/target/[@devel_project="YaST:Head"]' --filter-by './action/target[starts-with(@ring, "1")]' --filter-by '@id!="1234567"' --filter-by 'contains(description, "#Portus")' --group-by='./action/target/@devel_project' --group-by='./action/target/@ring' Multiple filter-by or group-by options may be used at the same time. Note that when using proposal mode, multiple stagings to consider may be provided in addition to a list of requests by which to filter. A more complex example: select --group-by='./action/target/@devel_project' A B C 123 456 789 This will separate the requests 123, 456, 789 by devel project and only consider stagings A, B, or C, if available, for placement. No arguments is also a valid choice and will propose all non-ignored requests into the first available staging. Note that bootstrapped stagings are only used when either required or no other stagings are available. Another useful example is placing all open requests into a specific letter staging with: select A Built in strategies may be specified as well. For example: select --strategy devel select --strategy quick select --strategy special select --strategy super The default is none and custom is used with any filter-by or group-by arguments are provided. To merge applicable requests into an existing staging. select --merge A To automatically try all available strategies. select --try-strategies These concepts can be combined and interactive mode allows the proposal to be modified before it is executed. Moving requests can be accomplished using the --move flag. For example, to move already staged pac1 and pac2 to staging B use the following. select --move B pac1 pac2 The staging in which the requests are staged will automatically be determined and the requests will be removed from that staging and placed in the specified staging. Related to this, the --filter-from option may be used in conjunction with --move to only move requests already staged in a specific staging. This can be useful if a staging master is responsible for a specific set of packages and wants to move them into a different staging when they were already placed in a mixed staging. For example, if one had a file with a list of packages the following would move any of them found in staging A to staging B. select --move --filter-from A B $(< package.list) "unselect" will remove from the project - pushing them back to the backlog If a message is included the requests will be ignored first. Use the --cleanup flag to include all obsolete requests. "unlock" will remove the staging lock in case it gets stuck or a manual hold If a command lock gets stuck while a hold is placed on a project the unlock command will need to be run twice since there are two layers of locks. "rebuild" will rebuild broken packages in the given stagings or all The rebuild command will only trigger builds for packages with less than 3 failures since the last success or if the build log indicates a stall. If the force option is included the rebuild checks will be ignored and all packages failing to build will be triggered. "setprio" will set priority of requests withing stagings If no stagings are specified all stagings will be used. The default priority is important, but the possible values are: "critical", "important", "moderate" or "low". "supersede" will supersede requests were applicable. A request list can be used to limit what is superseded. Usage: osc staging accept [--force] [--no-cleanup] [LETTER...] osc staging acheck osc staging adi [--move] [--by-develproject] [--split] [REQUEST...] osc staging check [--old] [STAGING...] osc staging check_duplicate_binaries osc staging config [--append] [--clear] [STAGING...] [key] [value] osc staging cleanup_rings osc staging freeze [--no-bootstrap] STAGING... osc staging frozenage [STAGING...] osc staging ignore [-m MESSAGE] REQUEST... osc staging unignore [--cleanup] [REQUEST...|all] osc staging list [--supersede] osc staging lock [-m MESSAGE] osc staging select [--no-freeze] [--move [--filter-from STAGING]] [--add PACKAGE] STAGING REQUEST... osc staging select [--no-freeze] [--interactive|--non-interactive] [--filter-by...] [--group-by...] [--merge] [--try-strategies] [--strategy] [STAGING...] [REQUEST...] osc staging unselect [--cleanup] [-m MESSAGE] [REQUEST...] osc staging unlock osc staging rebuild [--force] [STAGING...] osc staging repair [--cleanup] [REQUEST...] osc staging setprio [STAGING...] [priority] osc staging supersede [REQUEST...] """ if opts.version: self._print_version() # verify the argument counts match the commands if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help staging"!') cmd = args[0] if cmd in ( 'accept', 'adi', 'check', 'config', 'frozenage', 'unignore', 'select', 'unselect', 'rebuild', 'repair', 'setprio', 'supersede', ): min_args, max_args = 0, None elif cmd in ( 'freeze', 'ignore', ): min_args, max_args = 1, None elif cmd in ( 'acheck', 'check_duplicate_binaries', 'cleanup_rings', 'list', 'lock', 'unlock', ): min_args, max_args = 0, 0 else: raise oscerr.WrongArgs('Unknown command: %s' % cmd) args = clean_args(args) if len(args) - 1 < min_args: raise oscerr.WrongArgs('Too few arguments.') if max_args is not None and len(args) - 1 > max_args: raise oscerr.WrongArgs('Too many arguments.') # Allow for determining project from osc store. if not opts.project: if core.is_project_dir('.'): opts.project = core.store_read_project('.') else: opts.project = 'Factory' # Cache the remote config fetch. Cache.init() # Init the OBS access and configuration opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False Config(opts.apiurl, opts.project) colorama.init(autoreset=True, strip=(opts.no_color or not bool(int(conf.config.get('staging.color', True))))) # Allow colors to be changed. for name in dir(Fore): if not name.startswith('_'): # .oscrc requires keys to be lower-case. value = conf.config.get('staging.color.' + name.lower()) if value: setattr(Fore, name, ansi.code_to_chars(value)) if opts.wipe_cache: Cache.delete_all() api = StagingAPI(opts.apiurl, opts.project) needed = lock_needed(cmd, opts) with OBSLock(opts.apiurl, opts.project, reason=cmd, needed=needed) as lock: # call the respective command and parse args by need if cmd == 'check': if len(args) == 1: CheckCommand(api).perform(None, opts.old) else: for prj in args[1:]: CheckCommand(api).perform(prj, opts.old) print() elif cmd == 'check_duplicate_binaries': CheckDuplicateBinariesCommand(api).perform(opts.save) elif cmd == 'config': projects = set() key = value = None stagings = api.get_staging_projects_short(None) + \ api.get_staging_projects() for arg in args[1:]: if arg in stagings: projects.add(api.prj_from_short(arg)) elif key is None: key = arg elif value is None: value = arg else: value += ' ' + arg if not len(projects): projects = api.get_staging_projects() ConfigCommand(api).perform(projects, key, value, opts.append, opts.clear) elif cmd == 'freeze': for prj in args[1:]: prj = api.prj_from_short(prj) print(Fore.YELLOW + prj) FreezeCommand(api).perform(prj, copy_bootstrap=opts.bootstrap) elif cmd == 'frozenage': projects = api.get_staging_projects_short() if len(args) == 1 else args[1:] for prj in projects: prj = api.prj_from_letter(prj) print('{} last frozen {}{:.1f} days ago'.format( Fore.YELLOW + prj + Fore.RESET, Fore.GREEN if api.prj_frozen_enough(prj) else Fore.RED, api.days_since_last_freeze(prj))) elif cmd == 'acheck': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest: version_openqa = api.pseudometa_file_load('version_totest') totest_dirty = api.is_repo_dirty(api.project, 'totest') print("version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty)) else: print("acheck is unavailable in %s!\n" % (api.project)) elif cmd == 'accept': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest is None or opts.force: # SLE does not have a totest_version or openqa_version - ignore it version_openqa = version_totest totest_dirty = False else: version_openqa = api.pseudometa_file_load('version_totest') totest_dirty = api.is_repo_dirty(api.project, 'totest') if version_openqa == version_totest and not totest_dirty: cmd = AcceptCommand(api) for prj in args[1:]: if cmd.perform(api.prj_from_letter(prj), opts.force): cmd.reset_rebuild_data(prj) else: return if not opts.no_cleanup: if api.item_exists(api.prj_from_letter(prj)): cmd.cleanup(api.prj_from_letter(prj)) cmd.accept_other_new() if opts.project.startswith('openSUSE:'): cmd.update_factory_version() if api.item_exists(api.crebuild): cmd.sync_buildfailures() else: print("Not safe to accept: /totest is not yet synced") elif cmd == 'unselect': if opts.message: print('Ignoring requests first') IgnoreCommand(api).perform(args[1:], opts.message) UnselectCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'select': # Include list of all stagings in short-hand and by full name. existing_stagings = api.get_staging_projects_short(None) existing_stagings += api.get_staging_projects() stagings = [] requests = [] for arg in args[1:]: # Since requests may be given by either request ID or package # name and stagings may include multi-letter special stagings # there is no easy way to distinguish between stagings and # requests in arguments. Therefore, check if argument is in the # list of short-hand and full project name stagings, otherwise # consider it a request. This also allows for special stagings # with the same name as package, but the staging will be assumed # first time around. The current practice seems to be to start a # special staging with a capital letter which makes them unique. # lastly adi stagings are consistently prefix with adi: which # also makes it consistent to distinguish them from request IDs. if arg in existing_stagings and arg not in stagings: stagings.append(api.extract_staging_short(arg)) elif arg not in requests: requests.append(arg) if len(stagings) != 1 or len(requests) == 0 or opts.filter_by or opts.group_by: if opts.move or opts.filter_from: print('--move and --filter-from must be used with explicit staging and request list') return open_requests = api.get_open_requests({'withhistory': 1}) if len(open_requests) == 0: print('No open requests to consider') return splitter = RequestSplitter(api, open_requests, in_ring=True) considerable = splitter.stagings_load(stagings) if considerable == 0: print('No considerable stagings on which to act') return if opts.merge: splitter.merge() if opts.try_strategies: splitter.strategies_try() if len(requests) > 0: splitter.strategy_do('requests', requests=requests) if opts.strategy: splitter.strategy_do(opts.strategy) elif opts.filter_by or opts.group_by: kwargs = {} if opts.filter_by: kwargs['filters'] = opts.filter_by if opts.group_by: kwargs['groups'] = opts.group_by splitter.strategy_do('custom', **kwargs) else: if opts.merge: # Merge any none strategies before final none strategy. splitter.merge(strategy_none=True) splitter.strategy_do('none') splitter.strategy_do_non_bootstrapped('none') proposal = splitter.proposal if len(proposal) == 0: print('Empty proposal') return if opts.interactive: with tempfile.NamedTemporaryFile(suffix='.yml') as temp: temp.write(yaml.safe_dump(splitter.proposal, default_flow_style=False) + '\n\n') if len(splitter.requests): temp.write('# remaining requests:\n') for request in splitter.requests: temp.write('# {}: {}\n'.format( request.get('id'), request.find('action/target').get('package'))) temp.write('\n') temp.write('# move requests between stagings or comment/remove them\n') temp.write('# change the target staging for a group\n') temp.write('# remove the group, requests, staging, or strategy to skip\n') temp.write('# stagings\n') if opts.merge: temp.write('# - mergeable: {}\n' .format(', '.join(sorted(splitter.stagings_mergeable + splitter.stagings_mergeable_none)))) temp.write('# - considered: {}\n' .format(', '.join(sorted(splitter.stagings_considerable)))) temp.write('# - remaining: {}\n' .format(', '.join(sorted(splitter.stagings_available)))) temp.flush() editor = os.getenv('EDITOR') if not editor: editor = 'xdg-open' return_code = subprocess.call(editor.split(' ') + [temp.name]) proposal = yaml.safe_load(open(temp.name).read()) # Filter invalidated groups from proposal. keys = ['group', 'requests', 'staging', 'strategy'] for group, info in sorted(proposal.items()): for key in keys: if not info.get(key): del proposal[group] break print(yaml.safe_dump(proposal, default_flow_style=False)) print('Accept proposal? [y/n] (y): ', end='') if opts.non_interactive: print('y') else: response = raw_input().lower() if response != '' and response != 'y': print('Quit') return for group, info in sorted(proposal.items()): print('Staging {} in {}'.format(group, info['staging'])) # SelectCommand expects strings. request_ids = map(str, info['requests'].keys()) target_project = api.prj_from_short(info['staging']) if 'merge' not in info: # Assume that the original splitter_info is desireable # and that this staging is simply manual followup. api.set_splitter_info_in_prj_pseudometa(target_project, info['group'], info['strategy']) SelectCommand(api, target_project) \ .perform(request_ids, no_freeze=opts.no_freeze) else: target_project = api.prj_from_short(stagings[0]) if opts.add: api.mark_additional_packages(target_project, [opts.add]) else: SelectCommand(api, target_project) \ .perform(requests, opts.move, api.prj_from_short(opts.filter_from), opts.no_freeze) elif cmd == 'cleanup_rings': CleanupRings(api).perform() elif cmd == 'ignore': IgnoreCommand(api).perform(args[1:], opts.message) elif cmd == 'unignore': UnignoreCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'list': ListCommand(api).perform(supersede=opts.supersede) elif cmd == 'lock': lock.hold(opts.message) elif cmd == 'adi': AdiCommand(api).perform(args[1:], move=opts.move, by_dp=opts.by_develproject, split=opts.split) elif cmd == 'rebuild': RebuildCommand(api).perform(args[1:], opts.force) elif cmd == 'repair': RepairCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'setprio': stagings = [] priority = None priorities = ['critical', 'important', 'moderate', 'low'] for arg in args[1:]: if arg in priorities: priority = arg else: stagings.append(arg) PrioCommand(api).perform(stagings, priority) elif cmd == 'supersede': SupersedeCommand(api).perform(args[1:]) elif cmd == 'unlock': lock.release(force=True)
def perform(self, packages=None, supersede=False): """ Perform the list command """ if not packages: packages = [] if supersede: if packages: self.api.dispatch_open_requests(packages) else: # First dispatch all possible requests self.api.dispatch_open_requests() # Print out the left overs requests = self.api.get_open_requests() non_ring_packages = [] change_devel_requests = {} result = {} for request in requests: # Consolidate all data from request request_id = int(request.get('id')) action = request.findall('action') if not action: msg = 'Request {} has no action'.format(request_id) raise oscerr.WrongArgs(msg) # we care only about first action action = action[0] # Where are we targeting the package target_package = action.find('target').get('package') # handle change_devel requests if action.get('type') == 'change_devel': change_devel_requests[target_package] = request_id continue # If the system have rings, we ask for the ring of the # package if self.api.crings: ring = self.api.ring_packages_for_links.get(target_package) if ring: # cut off *:Rings: prefix ring = ring[len(self.api.crings) + 1:] else: ring = self.api.project # list all deletereq as in-ring if action.get('type') == 'delete': if ring: ring = ring + " (delete request)" else: ring = '(delete request)' # This condition is quite moot as we dispatched stuff # above anyway if ring: devel = self.api.get_devel_project("openSUSE:Factory", target_package) if devel is None: devel = '00' result.setdefault(devel, []).append('sr#{}: {:<30} -> {:<12}'.format( request_id, target_package, ring)) # show origin of request if self.api.project != "openSUSE:Factory" and action.find( 'source') != None: source_prj = action.find('source').get('project') if source_prj.startswith('SUSE:SLE-12:') \ or source_prj.startswith('SUSE:SLE-12-'): source_prj = source_prj[len('SUSE:SLE-12:'):] elif source_prj.startswith('openSUSE:'): source_prj = source_prj[len('openSUSE:'):] if source_prj.startswith('Leap:'): source_prj = source_prj[len('Leap:'):] elif source_prj.startswith('home:'): source_prj = '~' + source_prj[len('home:'):] result[devel][-1] += ' ({})'.format(source_prj) else: non_ring_packages.append(target_package) for prj in sorted(result.keys()): print prj for line in result[prj]: print ' ', line if len(non_ring_packages): print "Not in a ring:", ' '.join(sorted(non_ring_packages)) if len(change_devel_requests): print "\nChange devel requests:" for package, requestid in change_devel_requests.items(): print('Request({}): {}'.format( package, 'https://build.opensuse.org/request/show/' + str(requestid)))
def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects ${cmd_option_list} "accept" will accept all requests in openSUSE:Factory:Staging:<LETTER> (into Factory) "acheck" will check if it's safe to accept new staging projects As openSUSE:Factory is syncing the right package versions between /standard, /totest and /snapshot, it's important that the projects are clean prior to a checkin round. "check" will check if all packages are links without changes "cleanup_rings" will try to cleanup rings content and print out problems "freeze" will freeze the sources of the project's links (not affecting the packages actually in) "frozenage" will show when the respective staging project was last frozen "list" will pick the requests not in rings "select" will add requests to the project "unselect" will remove from the project - pushing them back to the backlog Usage: osc staging accept [--force] [LETTER...] osc staging check [--old] REPO osc staging cleanup_rings osc staging freeze [--no-boostrap] PROJECT... osc staging frozenage PROJECT... osc staging list [--supersede] osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST... osc staging unselect REQUEST... osc staging repair REQUEST... """ if opts.version: self._print_version() # verify the argument counts match the commands if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help staging"!') cmd = args[0] if cmd in ('freeze', 'frozenage', 'repair'): min_args, max_args = 1, None elif cmd == 'check': min_args, max_args = 0, 2 elif cmd == 'select': min_args, max_args = 1, None if not opts.add: min_args = 2 elif cmd == 'unselect': min_args, max_args = 1, None elif cmd == 'adi': min_args, max_args = None, None elif cmd in ('list', 'accept'): min_args, max_args = 0, None elif cmd in ('cleanup_rings', 'acheck'): min_args, max_args = 0, 0 else: raise oscerr.WrongArgs('Unknown command: %s' % cmd) if len(args) - 1 < min_args: raise oscerr.WrongArgs('Too few arguments.') if max_args is not None and len(args) - 1 > max_args: raise oscerr.WrongArgs('Too many arguments.') # Init the OBS access and configuration opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False Config(opts.project) with OBSLock(opts.apiurl, opts.project): api = StagingAPI(opts.apiurl, opts.project) # call the respective command and parse args by need if cmd == 'check': prj = args[1] if len(args) > 1 else None CheckCommand(api).perform(prj, opts.old) elif cmd == 'freeze': for prj in args[1:]: FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap=opts.bootstrap) elif cmd == 'frozenage': for prj in args[1:]: print "%s last frozen %0.1f days ago" % (api.prj_from_letter( prj), api.days_since_last_freeze(api.prj_from_letter(prj))) elif cmd == 'acheck': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') print "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % ( version_openqa, version_totest, totest_dirty) else: print "acheck is unavailable in %s!\n" % (api.project) elif cmd == 'accept': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest is None or opts.force: # SLE does not have a totest_version or openqa_version - ignore it version_openqa = version_totest totest_dirty = False else: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') if version_openqa == version_totest and not totest_dirty: cmd = AcceptCommand(api) for prj in args[1:]: if not cmd.perform(api.prj_from_letter(prj), opts.force): return if not opts.no_cleanup: if api.item_exists(api.prj_from_letter(prj)): cmd.cleanup(api.prj_from_letter(prj)) if api.item_exists("%s:DVD" % api.prj_from_letter(prj)): cmd.cleanup("%s:DVD" % api.prj_from_letter(prj)) if opts.project.startswith('openSUSE:'): cmd.accept_other_new() cmd.update_factory_version() if api.item_exists(api.crebuild): cmd.sync_buildfailures() else: print "Not safe to accept: /totest is not yet synced" elif cmd == 'unselect': UnselectCommand(api).perform(args[1:]) elif cmd == 'select': tprj = api.prj_from_letter(args[1]) if opts.add: api.mark_additional_packages(tprj, [opts.add]) else: SelectCommand(api, tprj).perform(args[2:], opts.move, opts.from_, opts.no_freeze) elif cmd == 'cleanup_rings': CleanupRings(api).perform() elif cmd == 'list': ListCommand(api).perform(args[1:], supersede=opts.supersede) elif cmd == 'adi': AdiCommand(api).perform(args[1:], move=opts.move, by_dp=opts.by_develproject, split=opts.split) elif cmd == 'repair': RepairCommand(api).perform(args[1:])
def do_overview(self, subcmd, opts, *args): """${cmd_name}: Overview of various repositories. For a full description, read: http://en.opensuse.org/openSUSE:OSC_overview_plugin overview viewname : will attempt to read the group from ~/.osc-overview/groupname.ini and display the data. Options: -c, --changelog display diff of changes across the newest and the previous versions of the packages. -p, --patchinfo Create a patchinfo template --color Colorize the output. Also 'color=1' in $group.ini. --no-color Do not colorize the output. Also 'color=0' in $group.ini. Usage: osc overview [Options] {group} You should define your groups in ~/.osc-overview/$group.ini """ if not os.path.exists(os.path.expanduser("~/.osc-overview")): print("Drop your views in ~/.osc-overview") exit(1) sys.path.append(os.path.expanduser('~/.osc-plugins')) cmds = [] cfgfiles = os.listdir(os.path.expanduser('~/.osc-overview')) for cfg in cfgfiles: cmds.append(os.path.basename(cfg.replace('.ini',''))) if not args or not os.path.exists(os.path.expanduser('~/.osc-overview/%s.ini' % args[0]) ): raise oscerr.WrongArgs('Unknown action. Choose one of: %s.' \ % ', '.join(cmds)) cmd = args[0] min_args, max_args = 0, 0 if len(args) - 1 < min_args: raise oscerr.WrongArgs('Too few arguments.') if len(args) - 1 > max_args: raise oscerr.WrongArgs('Too many arguments.') from oscpluginoverview.sources import GemSource, BuildServiceSource, BuildServicePendingRequestsSource #gems = GemSource("foo") #print(gems.packages()) #print(gems.version('rubygem-hpricot')) #obs = BuildServiceSource('http://api.opensuse.org', 'zypp:Head') #print(obs.changelog('libzypp')) #print(obs.packages()) #print(obs.version('libzypp')) #reqs = BuildServicePendingRequestsSource('http://api.opensuse.org', 'openSUSE:Factory') #print(reqs.packages()) #print(reqs.version("patch")) self._overview(cmd, opts)
def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects ${cmd_option_list} "accept" will accept all requests in $PROJECT:Staging:<LETTER> into $PROJECT If openSUSE:* project, requests marked ready from adi stagings will also be accepted. "acheck" will check if it is safe to accept new staging projects As $PROJECT is syncing the right package versions between /standard, /totest and /snapshot, it is important that the projects are clean prior to a checkin round. "adi" will list already staged requests, stage new requests, and supersede requests where applicable. New adi stagings will be created for new packages based on the grouping options used. The default grouping is by source project. When adi stagings are ready the request will be marked ready, unstaged, and the adi staging deleted. "check" will check if all packages are links without changes "cleanup_rings" will try to cleanup rings content and print out problems "freeze" will freeze the sources of the project's links while not affecting the source packages "frozenage" will show when the respective staging project was last frozen "ignore" will ignore a request from "list" and "adi" commands until unignored "unignore" will remove from requests from ignore list If the --cleanup flag is included then all ignored requests that were changed from state new or review more than 3 days ago will be removed. "list" will list/supersede requests for ring packages or all if no rings. The package list is used to limit what requests are superseded when called with the --supersede option. "repair" will attempt to repair the state of a request that has been corrupted. Use the --cleanup flag to include all untracked requests. "select" will add requests to the project Stagings are expected to be either in short-hand or the full project name. For example letter or named stagings can be specified simply as A, B, Gcc6, etc, while adi stagings can be specified as adi:1, adi:2, etc. Currently, adi stagings are not supported in proposal mode. Requests may either be the target package or the request ID. When using --filter-by or --group-by the xpath will be applied to the request node as returned by OBS. Several values will supplement the normal request node. - ./action/target/@devel_project: the devel project for the package - ./action/target/@ring: the ring to which the package belongs - ./@ignored: either false or the provided message Some useful examples: --filter-by './action/target[starts-with(@package, "yast-")]' --filter-by './action/target/[@devel_project="YaST:Head"]' --filter-by './action/target[starts-with(@ring, "1")]' --filter-by '@id!="1234567"' --group-by='./action/target/@devel_project' --group-by='./action/target/@ring' Multiple filter-by or group-by options may be used at the same time. Note that when using proposal mode, multiple stagings to consider may be provided in addition to a list of requests by which to filter. A more complex example: select --group-by='./action/target/@devel_project' A B C 123 456 789 This will separate the requests 123, 456, 789 by devel project and only consider stagings A, B, or C, if available, for placement. No arguments is also a valid choice and will propose all non-ignored requests into the first available staging. Note that bootstrapped stagings are only used when either required or no other stagings are available. Another useful example is placing all open requests into a specific letter staging with: select A Interactive mode allows the proposal to be modified before application. "unselect" will remove from the project - pushing them back to the backlog If a message is included the requests will be ignored first. Use the --cleanup flag to include all obsolete requests. "unlock" will remove the staging lock in case it gets stuck "rebuild" will rebuild broken packages in the given stagings or all The rebuild command will only trigger builds for packages with less than 3 failures since the last success or if the build log indicates a stall. If the force option is included the rebuild checks will be ignored and all packages failing to build will be triggered. Usage: osc staging accept [--force] [--no-cleanup] [LETTER...] osc staging acheck osc staging adi [--move] [--by-develproject] [--split] [REQUEST...] osc staging check [--old] STAGING osc staging cleanup_rings osc staging freeze [--no-boostrap] STAGING... osc staging frozenage [STAGING...] osc staging ignore [-m MESSAGE] REQUEST... osc staging unignore [--cleanup] [REQUEST...|all] osc staging list [--supersede] [PACKAGE...] osc staging select [--no-freeze] [--move [--from STAGING]] [--add PACKAGE] STAGING REQUEST... osc staging select [--no-freeze] [--interactive|--non-interactive] [--filter-by...] [--group-by...] [--merge] [--try-strategies] [--strategy] [STAGING...] [REQUEST...] osc staging unselect [--cleanup] [-m MESSAGE] [REQUEST...] osc staging unlock osc staging rebuild [--force] [STAGING...] osc staging repair [--cleanup] [REQUEST...] osc staging setprio [STAGING...] """ if opts.version: self._print_version() # verify the argument counts match the commands if len(args) == 0: raise oscerr.WrongArgs('No command given, see "osc help staging"!') cmd = args[0] if cmd == 'freeze': min_args, max_args = 1, None elif cmd == 'repair': min_args, max_args = 0, None elif cmd == 'frozenage': min_args, max_args = 0, None elif cmd == 'setprio': min_args, max_args = 0, None elif cmd == 'check': min_args, max_args = 0, 1 elif cmd == 'select': min_args, max_args = 0, None elif cmd == 'unselect': min_args, max_args = 0, None elif cmd == 'adi': min_args, max_args = 0, None elif cmd == 'ignore': min_args, max_args = 1, None elif cmd == 'unignore': min_args, max_args = 0, None elif cmd in ('list', 'accept'): min_args, max_args = 0, None elif cmd in ('cleanup_rings', 'acheck'): min_args, max_args = 0, 0 elif cmd == 'unlock': min_args, max_args = 0, 0 elif cmd == 'rebuild': min_args, max_args = 0, None else: raise oscerr.WrongArgs('Unknown command: %s' % cmd) if len(args) - 1 < min_args: raise oscerr.WrongArgs('Too few arguments.') if max_args is not None and len(args) - 1 > max_args: raise oscerr.WrongArgs('Too many arguments.') # Init the OBS access and configuration opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False Config(opts.project) if opts.wipe_cache: Cache.delete_all() lock = OBSLock(opts.apiurl, opts.project) if cmd == 'unlock': lock.release() return with lock: api = StagingAPI(opts.apiurl, opts.project) # call the respective command and parse args by need if cmd == 'check': prj = args[1] if len(args) > 1 else None CheckCommand(api).perform(prj, opts.old) elif cmd == 'freeze': for prj in args[1:]: FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap=opts.bootstrap) elif cmd == 'frozenage': projects = api.get_staging_projects_short() if len( args) == 1 else args[1:] for prj in projects: print("%s last frozen %0.1f days ago" % (api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj)))) elif cmd == 'acheck': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') print( "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty)) else: print("acheck is unavailable in %s!\n" % (api.project)) elif cmd == 'accept': # Is it safe to accept? Meaning: /totest contains what it should and is not dirty version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64") if version_totest is None or opts.force: # SLE does not have a totest_version or openqa_version - ignore it version_openqa = version_totest totest_dirty = False else: version_openqa = api.load_file_content( "%s:Staging" % api.project, "dashboard", "version_totest") totest_dirty = api.is_repo_dirty(api.project, 'totest') if version_openqa == version_totest and not totest_dirty: cmd = AcceptCommand(api) for prj in args[1:]: if cmd.perform(api.prj_from_letter(prj), opts.force): cmd.reset_rebuild_data(prj) else: return if not opts.no_cleanup: if api.item_exists(api.prj_from_letter(prj)): cmd.cleanup(api.prj_from_letter(prj)) if api.item_exists("%s:DVD" % api.prj_from_letter(prj)): cmd.cleanup("%s:DVD" % api.prj_from_letter(prj)) if opts.project.startswith('openSUSE:'): cmd.accept_other_new() cmd.update_factory_version() if api.item_exists(api.crebuild): cmd.sync_buildfailures() else: print("Not safe to accept: /totest is not yet synced") elif cmd == 'unselect': if opts.message: print('Ignoring requests first') IgnoreCommand(api).perform(args[1:], opts.message) UnselectCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'select': # Include list of all stagings in short-hand and by full name. existing_stagings = api.get_staging_projects_short(None) existing_stagings += [ p for p in api.get_staging_projects() if not p.endswith(':DVD') ] stagings = [] requests = [] for arg in args[1:]: # Since requests may be given by either request ID or package # name and stagings may include multi-letter special stagings # there is no easy way to distinguish between stagings and # requests in arguments. Therefore, check if argument is in the # list of short-hand and full project name stagings, otherwise # consider it a request. This also allows for special stagings # with the same name as package, but the staging will be assumed # first time around. The current practice seems to be to start a # special staging with a capital letter which makes them unique. # lastly adi stagings are consistently prefix with adi: which # also makes it consistent to distinguish them from request IDs. if arg in existing_stagings and arg not in stagings: stagings.append(api.extract_staging_short(arg)) elif arg not in requests: requests.append(arg) if len(stagings) != 1 or len( requests) == 0 or opts.filter_by or opts.group_by: if opts.move or opts.from_: print( '--move and --from must be used with explicit staging and request list' ) return open_requests = api.get_open_requests() if len(open_requests) == 0: print('No open requests to consider') return splitter = RequestSplitter(api, open_requests, in_ring=True) considerable = splitter.stagings_load(stagings) if considerable == 0: print('No considerable stagings on which to act') return if opts.merge: splitter.merge() if opts.try_strategies: splitter.strategies_try() if len(requests) > 0: splitter.strategy_do('requests', requests=requests) if opts.strategy: splitter.strategy_do(opts.strategy) elif opts.filter_by or opts.group_by: kwargs = {} if opts.filter_by: kwargs['filters'] = opts.filter_by if opts.group_by: kwargs['groups'] = opts.group_by splitter.strategy_do('custom', **kwargs) else: if opts.merge: # Merge any none strategies before final none strategy. splitter.merge(strategy_none=True) splitter.strategy_do('none') splitter.strategy_do_non_bootstrapped('none') proposal = splitter.proposal if len(proposal) == 0: print('Empty proposal') return if opts.interactive: with tempfile.NamedTemporaryFile(suffix='.yml') as temp: temp.write( yaml.safe_dump(splitter.proposal, default_flow_style=False) + '\n\n') temp.write( '# move requests between stagings or comment/remove them\n' ) temp.write('# change the target staging for a group\n') temp.write('# stagings\n') if opts.merge: temp.write('# - merged: {}\n'.format(', '.join( sorted(splitter.stagings_mergeable + splitter.stagings_mergeable_none)))) temp.write('# - considered: {}\n'.format(', '.join( sorted(splitter.stagings_considerable)))) temp.write('# - remaining: {}\n'.format(', '.join( sorted(splitter.stagings_available)))) temp.flush() editor = os.getenv('EDITOR') if not editor: editor = 'xdg-open' return_code = subprocess.call([editor, temp.name]) proposal = yaml.safe_load(open(temp.name).read()) # Filter invalidated groups from proposal. keys = ['group', 'requests', 'staging', 'strategy'] for group, info in sorted(proposal.items()): for key in keys: if not info.get(key): del proposal[group] break print(yaml.safe_dump(proposal, default_flow_style=False)) print('Accept proposal? [y/n] (y): ', end='') if opts.non_interactive: print('y') else: response = raw_input().lower() if response != '' and response != 'y': print('Quit') return for group, info in sorted(proposal.items()): print('Staging {} in {}'.format(group, info['staging'])) # SelectCommand expects strings. request_ids = map(str, info['requests'].keys()) target_project = api.prj_from_short(info['staging']) if 'merge' not in info: # Assume that the original splitter_info is desireable # and that this staging is simply manual followup. api.set_splitter_info_in_prj_pseudometa( target_project, info['group'], info['strategy']) SelectCommand(api, target_project) \ .perform(request_ids, no_freeze=opts.no_freeze) else: target_project = api.prj_from_short(stagings[0]) if opts.add: api.mark_additional_packages(target_project, [opts.add]) else: SelectCommand(api, target_project) \ .perform(requests, opts.move, opts.from_, opts.no_freeze) elif cmd == 'cleanup_rings': CleanupRings(api).perform() elif cmd == 'ignore': IgnoreCommand(api).perform(args[1:], opts.message) elif cmd == 'unignore': UnignoreCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'list': ListCommand(api).perform(args[1:], supersede=opts.supersede) elif cmd == 'adi': AdiCommand(api).perform(args[1:], move=opts.move, by_dp=opts.by_develproject, split=opts.split) elif cmd == 'rebuild': RebuildCommand(api).perform(args[1:], opts.force) elif cmd == 'repair': RepairCommand(api).perform(args[1:], opts.cleanup) elif cmd == 'setprio': PrioCommand(api).perform(args[1:])