def create_new_adi(self, wanted_requests, split=False):
source_projects_expand = self.config.get('source_projects_expand',
'').split()
# if we don't call it, there is no invalidate function added
requests = self.api.get_open_requests()
if len(wanted_requests):
requests = []
rf = RequestFinder(self.api)
for p in wanted_requests:
requests.append(rf.load_request(p))
splitter = RequestSplitter(self.api, requests, in_ring=False)
splitter.filter_add('./action[@type="submit" or @type="delete"]')
if len(wanted_requests):
splitter.filter_add_requests([str(p) for p in wanted_requests])
# wanted_requests forces all requests into a single group.
else:
if split:
splitter.group_by('./@id')
else:
splitter.group_by('./action/source/@project')
splitter.split()
for group in sorted(splitter.grouped.keys()):
print(Fore.YELLOW + (group if group != '' else 'wanted') +
Fore.RESET)
name = None
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
target = request.find('./action/target')
target_package = target.get('package')
line = '- {} {}{:<30}{}'.format(request_id, Fore.CYAN,
target_package, Fore.RESET)
message = self.api.ignore_format(request_id)
if message:
print(line + '\n' + Fore.WHITE + message + Fore.RESET)
continue
# Auto-superseding request in adi command
stage_info, code = self.api.update_superseded_request(request)
if stage_info:
print(line +
' ({})'.format(SupersedeCommand.CODE_MAP[code]))
continue
# Only create staging projec the first time a non superseded
# request is processed from a particular group.
if name is None:
use_frozenlinks = group in source_projects_expand and not split
name = self.api.create_adi_project(None, use_frozenlinks,
group)
if not self.api.rq_to_prj(request_id, name):
return False
print(line + Fore.GREEN + ' (staged in {})'.format(name) +
Fore.RESET)
def create_new_adi(self, wanted_requests, by_dp=False, split=False):
source_projects_expand = self.config.get('source_projects_expand', '').split()
requests = self.api.get_open_requests()
splitter = RequestSplitter(self.api, requests, in_ring=False)
splitter.filter_add('./action[@type="submit"]')
if len(wanted_requests):
splitter.filter_add_requests([str(p) for p in wanted_requests])
# wanted_requests forces all requests into a single group.
else:
if split:
splitter.group_by('./@id')
elif by_dp:
splitter.group_by('./action/target/@devel_project')
else:
splitter.group_by('./action/source/@project')
if not split:
splitter.group_by('./action/target/@nonfree')
splitter.split()
for group in sorted(splitter.grouped.keys()):
print(Fore.YELLOW + (group if group != '' else 'wanted') + Fore.RESET)
name = None
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
target = request.find('./action/target')
target_package = target.get('package')
line = '- {} {}{:<30}{}'.format(request_id, Fore.CYAN, target_package, Fore.RESET)
message = self.api.ignore_format(request_id)
if message:
print(line + '\n' + Fore.WHITE + message + Fore.RESET)
continue
# Auto-superseding request in adi command
stage_info, code = self.api.update_superseded_request(request)
if stage_info:
print(line + ' ({})'.format(SupersedeCommand.CODE_MAP[code]))
continue
# Only create staging projec the first time a non superseded
# request is processed from a particular group.
if name is None:
use_frozenlinks = group in source_projects_expand and not split
nonfree = bool(target.get('nonfree'))
name = self.api.create_adi_project(None,
use_frozenlinks, group, nonfree)
if not self.api.rq_to_prj(request_id, name):
return False
print(line + Fore.GREEN + ' (staged in {})'.format(name) + Fore.RESET)
if name:
# Notify everybody about the changes.
self.api.update_status_comments(name, 'select')
def create_new_adi(self, wanted_requests, by_dp=False, split=False):
source_projects_expand = self.config.get('source_projects_expand', '').split()
requests = self.api.get_open_requests()
splitter = RequestSplitter(self.api, requests, in_ring=False)
splitter.filter_add('./action[@type="submit"]')
if len(wanted_requests):
splitter.filter_add_requests([str(p) for p in wanted_requests])
# wanted_requests forces all requests into a single group.
else:
if split:
splitter.group_by('./@id')
elif by_dp:
splitter.group_by('./action/target/@devel_project')
else:
splitter.group_by('./action/source/@project')
if not split:
splitter.group_by('./action/target/@nonfree')
splitter.split()
for group in sorted(splitter.grouped.keys()):
print(Fore.YELLOW + (group if group != '' else 'wanted') + Fore.RESET)
name = None
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
target = request.find('./action/target')
target_package = target.get('package')
line = '- {} {}{:<30}{}'.format(request_id, Fore.CYAN, target_package, Fore.RESET)
message = self.api.ignore_format(request_id)
if message:
print(line + '\n' + Fore.WHITE + message + Fore.RESET)
continue
# Auto-superseding request in adi command
stage_info, code = self.api.update_superseded_request(request)
if stage_info:
print(line + ' ({})'.format(SupersedeCommand.CODE_MAP[code]))
continue
# Only create staging projec the first time a non superseded
# request is processed from a particular group.
if name is None:
use_frozenlinks = group in source_projects_expand and not split
nonfree = bool(target.get('nonfree'))
name = self.api.create_adi_project(None,
use_frozenlinks, group, nonfree)
if not self.api.rq_to_prj(request_id, name):
return False
print(line + Fore.GREEN + ' (staged in {})'.format(name) + Fore.RESET)
if name:
# Notify everybody about the changes.
self.api.update_status_comments(name, 'select')
def perform(self, packages=None, supersede=False):
"""
Perform the list command
"""
if supersede:
self.api.dispatch_open_requests(packages)
requests = self.api.get_open_requests()
requests_ignored = self.api.get_ignored_requests()
splitter = RequestSplitter(self.api, requests, in_ring=True)
splitter.filter_add('./action[@type="change_devel"]')
change_devel_requests = splitter.filter_only()
splitter.reset()
splitter.filter_add('./action[not(@type="add_role" or @type="change_devel")]')
splitter.group_by('./action/target/@devel_project')
splitter.split()
is_factory = self.api.project != 'openSUSE:Factory'
for group in sorted(splitter.grouped.keys()):
print group
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
action = request.find('action')
target_package = action.find('target').get('package')
ring = action.find('target').get('ring')
if action.get('type') == 'delete':
ring += ' (delete request)'
line = 'sr#{}: {:<30} -> {:<12}'.format(request_id, target_package, ring)
if is_factory and action.find('source') != None:
source_project = action.find('source').get('project')
source_project = self.project_strip(source_project)
line += ' ({})'.format(source_project)
if request_id in requests_ignored:
line += '\n ignored: ' + str(requests_ignored[request_id])
print ' ', line
if len(splitter.other):
non_ring_packages = []
for request in splitter.other:
non_ring_packages.append(request.find('./action/target').get('package'))
print 'Not in a ring:', ' '.join(sorted(non_ring_packages))
if len(change_devel_requests):
print '\nChange devel requests:'
for request in change_devel_requests:
target_package = request.find('./action/target').get('package')
url = self.api.makeurl(['request', 'show', request.get('id')])
print('- request({}): {}'.format(target_package, url))
def create_new_adi(self, wanted_requests, by_dp=False, split=False):
requests = self.api.get_open_requests()
splitter = RequestSplitter(self.api, requests, in_ring=False)
splitter.filter_add('./action[@type="submit"]')
if len(wanted_requests):
splitter.filter_add_requests([str(p) for p in wanted_requests])
# wanted_requests forces all requests into a single group.
else:
if split:
splitter.group_by('./@id')
elif by_dp:
splitter.group_by('./action/target/@devel_project')
else:
splitter.group_by('./action/source/@project')
splitter.split()
for group in sorted(splitter.grouped.keys()):
print(group if group != '' else 'wanted')
name = None
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
target_package = request.find('./action/target').get('package')
line = '- sr#{}: {:<30}'.format(request_id, target_package)
if request_id in self.requests_ignored:
print(line + '\n ignored: ' +
str(self.requests_ignored[request_id]))
continue
# Auto-superseding request in adi command
if self.api.update_superseded_request(request):
print(line + ' (superseded)')
continue
# Only create staging projec the first time a non superseded
# request is processed from a particular group.
if name is None:
name = self.api.create_adi_project(None)
if not self.api.rq_to_prj(request_id, name):
return False
print(line + ' (staged in {})'.format(name))
if name:
# Notify everybody about the changes.
self.api.update_status_comments(name, 'select')
def perform(self, supersede=False):
"""
Perform the list command
"""
if supersede:
SupersedeCommand(self.api).perform()
requests = self.api.get_open_requests()
if not len(requests): return
splitter = RequestSplitter(self.api, requests, in_ring=True)
splitter.group_by('./action/target/@devel_project')
splitter.split()
hide_source = self.api.project == 'openSUSE:Factory'
for group in sorted(splitter.grouped.keys()):
print(Fore.YELLOW + group)
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
action = request.find('action')
target_package = action.find('target').get('package')
ring = action.find('target').get('ring')
ring_color = Fore.MAGENTA if ring.startswith('0') else ''
line = '{} {}{:<30}{} -> {}{:<12}{}'.format(
request_id, Fore.CYAN, target_package, Fore.RESET,
ring_color, ring, Fore.RESET)
if not hide_source and action.find('source') != None:
source_project = action.find('source').get('project')
source_project = self.project_strip(source_project)
line += ' ({})'.format(Fore.YELLOW + source_project + Fore.RESET)
if action.get('type') == 'delete':
line += ' (' + Fore.RED + 'delete request' + Fore.RESET + ')'
message = self.api.ignore_format(request_id)
if message:
line += '\n' + Fore.WHITE + message + Fore.RESET
print(' ' + line)
if len(splitter.other):
non_ring_packages = []
for request in splitter.other:
non_ring_packages.append(request.find('./action/target').get('package'))
print('Not in a ring: ' + ' '.join(sorted(non_ring_packages)))
# Print requests not handled by staging process to highlight them.
splitter.stageable = False
for request_type in ('change_devel', 'set_bugowner'):
splitter.reset()
splitter.filter_add('./action[@type="{}"]'.format(request_type))
requests = splitter.filter_only()
if len(requests):
print('\n{} request(s)'.format(request_type))
for request in sorted(requests, key=lambda s: s.get('id')):
print(' {} {}'.format(
self.api.makeurl(['request', 'show', request.get('id')]),
request.find('./action/target').get('package')))
def perform(self, supersede=False):
"""
Perform the list command
"""
if supersede:
SupersedeCommand(self.api).perform()
requests = self.api.get_open_requests()
if not len(requests): return
splitter = RequestSplitter(self.api, requests, in_ring=True)
splitter.filter_add('./action[@type="submit" or @type="delete"]')
splitter.group_by('./action/target/@devel_project')
splitter.split()
hide_source = self.api.project == 'openSUSE:Factory'
for group in sorted(splitter.grouped.keys()):
print Fore.YELLOW + group
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
action = request.find('action')
target_package = action.find('target').get('package')
ring = action.find('target').get('ring')
ring_color = Fore.MAGENTA if ring.startswith('0') else ''
line = '{} {}{:<30}{} -> {}{:<12}{}'.format(
request_id, Fore.CYAN, target_package, Fore.RESET,
ring_color, ring, Fore.RESET)
if not hide_source and action.find('source') != None:
source_project = action.find('source').get('project')
source_project = self.project_strip(source_project)
line += ' ({})'.format(Fore.YELLOW + source_project +
Fore.RESET)
if action.get('type') == 'delete':
line += ' (' + Fore.RED + 'delete request' + Fore.RESET + ')'
message = self.api.ignore_format(request_id)
if message:
line += '\n' + Fore.WHITE + message + Fore.RESET
print ' ', line
if len(splitter.other):
non_ring_packages = []
for request in splitter.other:
non_ring_packages.append(
request.find('./action/target').get('package'))
print 'Not in a ring:', ' '.join(sorted(non_ring_packages))
# Print requests not handled by staging process to highlight them.
for request_type in ('change_devel', 'set_bugowner'):
splitter.reset()
splitter.filter_add('./action[@type="{}"]'.format(request_type))
requests = splitter.filter_only()
if len(requests):
print('\n{} request(s)'.format(request_type))
for request in sorted(requests, key=lambda s: s.get('id')):
print(' {} {}'.format(
self.api.makeurl(
['request', 'show',
request.get('id')]),
request.find('./action/target').get('package')))
def do_staging(self, subcmd, opts, *args):
"""${cmd_name}: Commands to work with staging projects
${cmd_option_list}
"accept" will accept all requests in
$PROJECT:Staging:<LETTER> into $PROJECT
If openSUSE:* project, requests marked ready from adi stagings will also
be accepted.
"acheck" will check if it is safe to accept new staging projects
As $PROJECT is syncing the right package versions between
/standard, /totest and /snapshot, it is important that the projects
are clean prior to a checkin round.
"adi" will list already staged requests, stage new requests, and supersede
requests where applicable. New adi stagings will be created for new
packages based on the grouping options used. The default grouping is by
source project. When adi stagings are ready the request will be marked
ready, unstaged, and the adi staging deleted.
"check" will check if all packages are links without changes
"cleanup_rings" will try to cleanup rings content and print
out problems
"freeze" will freeze the sources of the project's links while not
affecting the source packages
"frozenage" will show when the respective staging project was last frozen
"ignore" will ignore a request from "list" and "adi" commands until unignored
"unignore" will remove from requests from ignore list
If the --cleanup flag is included then all ignored requests that were
changed from state new or review more than 3 days ago will be removed.
"list" will list/supersede requests for ring packages or all if no rings.
The package list is used to limit what requests are superseded when
called with the --supersede option.
"repair" will attempt to repair the state of a request that has been
corrupted.
"select" will add requests to the project
Stagings are expected to be either in short-hand or the full project
name. For example letter or named stagings can be specified simply as
A, B, Gcc6, etc, while adi stagings can be specified as adi:1, adi:2,
etc. Currently, adi stagings are not supported in proposal mode.
Requests may either be the target package or the request ID.
When using --filter-by or --group-by the xpath will be applied to the
request node as returned by OBS. Several values will supplement the
normal request node.
- ./action/target/@devel_project: the devel project for the package
- ./action/target/@ring: the ring to which the package belongs
- ./@ignored: either false or the provided message
Some useful examples:
--filter-by './action/target[starts-with(@package, "yast-")]'
--filter-by './action/target/[@devel_project="YaST:Head"]'
--filter-by './action/target[starts-with(@ring, "1")]'
--filter-by '@id!="1234567"'
--group-by='./action/target/@devel_project'
--group-by='./action/target/@ring'
Multiple filter-by or group-by options may be used at the same time.
Note that when using proposal mode, multiple stagings to consider may be
provided in addition to a list of requests by which to filter. A more
complex example:
select --group-by='./action/target/@devel_project' A B C 123 456 789
This will separate the requests 123, 456, 789 by devel project and only
consider stagings A, B, or C, if available, for placement.
No arguments is also a valid choice and will propose all non-ignored
requests into the first available staging. Note that bootstrapped
stagings are only used when either required or no other stagings are
available.
Another useful example is placing all open requests into a specific
letter staging with:
select A
Interactive mode allows the proposal to be modified before application.
"unselect" will remove from the project - pushing them back to the backlog
"unlock" will remove the staging lock in case it gets stuck
Usage:
osc staging accept [--force] [--no-cleanup] [LETTER...]
osc staging acheck
osc staging adi [--move] [--by-develproject] [--split] REQUEST...
osc staging check [--old] REPO
osc staging cleanup_rings
osc staging freeze [--no-boostrap] PROJECT...
osc staging frozenage PROJECT...
osc staging ignore [-m MESSAGE] REQUEST...
osc staging unignore [--cleanup] REQUEST...|all
osc staging list [--supersede] [PACKAGE...]
osc staging select [--no-freeze] [--move [--from PROJECT] STAGING REQUEST...
osc staging select [--no-freeze] [[--interactive] [--filter-by...] [--group-by...]] [STAGING...] [REQUEST...]
osc staging unselect REQUEST...
osc staging unlock
osc staging repair REQUEST...
"""
if opts.version:
self._print_version()
# verify the argument counts match the commands
if len(args) == 0:
raise oscerr.WrongArgs('No command given, see "osc help staging"!')
cmd = args[0]
if cmd in ('freeze', 'frozenage', 'repair'):
min_args, max_args = 1, None
elif cmd == 'check':
min_args, max_args = 0, 1
elif cmd == 'select':
min_args, max_args = 0, None
elif cmd == 'unselect':
min_args, max_args = 1, None
elif cmd == 'adi':
min_args, max_args = 0, None
elif cmd == 'ignore':
min_args, max_args = 1, None
elif cmd == 'unignore':
min_args, max_args = 0, None
elif cmd in ('list', 'accept'):
min_args, max_args = 0, None
elif cmd in ('cleanup_rings', 'acheck'):
min_args, max_args = 0, 0
elif cmd == 'unlock':
min_args, max_args = 0, 0
else:
raise oscerr.WrongArgs('Unknown command: %s' % cmd)
if len(args) - 1 < min_args:
raise oscerr.WrongArgs('Too few arguments.')
if max_args is not None and len(args) - 1 > max_args:
raise oscerr.WrongArgs('Too many arguments.')
# Init the OBS access and configuration
opts.project = self._full_project_name(opts.project)
opts.apiurl = self.get_api_url()
opts.verbose = False
Config(opts.project)
if opts.wipe_cache:
Cache.delete_all()
lock = OBSLock(opts.apiurl, opts.project)
if cmd == 'unlock':
lock.release()
return
with lock:
api = StagingAPI(opts.apiurl, opts.project)
# call the respective command and parse args by need
if cmd == 'check':
prj = args[1] if len(args) > 1 else None
CheckCommand(api).perform(prj, opts.old)
elif cmd == 'freeze':
for prj in args[1:]:
FreezeCommand(api).perform(api.prj_from_letter(prj),
copy_bootstrap=opts.bootstrap)
elif cmd == 'frozenage':
for prj in args[1:]:
print("%s last frozen %0.1f days ago" %
(api.prj_from_letter(prj),
api.days_since_last_freeze(api.prj_from_letter(prj))))
elif cmd == 'acheck':
# Is it safe to accept? Meaning: /totest contains what it should and is not dirty
version_totest = api.get_binary_version(api.project,
"openSUSE-release.rpm",
repository="totest",
arch="x86_64")
if version_totest:
version_openqa = api.load_file_content(
"%s:Staging" % api.project, "dashboard", "version_totest")
totest_dirty = api.is_repo_dirty(api.project, 'totest')
print(
"version_openqa: %s / version_totest: %s / totest_dirty: %s\n"
% (version_openqa, version_totest, totest_dirty))
else:
print("acheck is unavailable in %s!\n" % (api.project))
elif cmd == 'accept':
# Is it safe to accept? Meaning: /totest contains what it should and is not dirty
version_totest = api.get_binary_version(api.project,
"openSUSE-release.rpm",
repository="totest",
arch="x86_64")
if version_totest is None or opts.force:
# SLE does not have a totest_version or openqa_version - ignore it
version_openqa = version_totest
totest_dirty = False
else:
version_openqa = api.load_file_content(
"%s:Staging" % api.project, "dashboard", "version_totest")
totest_dirty = api.is_repo_dirty(api.project, 'totest')
if version_openqa == version_totest and not totest_dirty:
cmd = AcceptCommand(api)
for prj in args[1:]:
if not cmd.perform(api.prj_from_letter(prj), opts.force):
return
if not opts.no_cleanup:
if api.item_exists(api.prj_from_letter(prj)):
cmd.cleanup(api.prj_from_letter(prj))
if api.item_exists("%s:DVD" %
api.prj_from_letter(prj)):
cmd.cleanup("%s:DVD" % api.prj_from_letter(prj))
if opts.project.startswith('openSUSE:'):
cmd.accept_other_new()
cmd.update_factory_version()
if api.item_exists(api.crebuild):
cmd.sync_buildfailures()
else:
print("Not safe to accept: /totest is not yet synced")
elif cmd == 'unselect':
UnselectCommand(api).perform(args[1:])
elif cmd == 'select':
# Include list of all stagings in short-hand and by full name.
existing_stagings = api.get_staging_projects_short(None)
existing_stagings += [
p for p in api.get_staging_projects() if not p.endswith(':DVD')
]
stagings = []
requests = []
for arg in args[1:]:
# Since requests may be given by either request ID or package
# name and stagings may include multi-letter special stagings
# there is no easy way to distinguish between stagings and
# requests in arguments. Therefore, check if argument is in the
# list of short-hand and full project name stagings, otherwise
# consider it a request. This also allows for special stagings
# with the same name as package, but the staging will be assumed
# first time around. The current practice seems to be to start a
# special staging with a capital letter which makes them unique.
# lastly adi stagings are consistently prefix with adi: which
# also makes it consistent to distinguish them from request IDs.
if arg in existing_stagings and arg not in stagings:
stagings.append(api.extract_staging_short(arg))
elif arg not in requests:
requests.append(arg)
if len(stagings) != 1 or len(
requests) == 0 or opts.filter_by or opts.group_by:
if opts.move or opts.from_:
print(
'--move and --from must be used with explicit staging and request list'
)
return
splitter = RequestSplitter(api,
api.get_open_requests(),
in_ring=True)
if len(requests) > 0:
splitter.filter_add_requests(requests)
if len(splitter.filters) == 0:
splitter.filter_add(
'./action[not(@type="add_role" or @type="change_devel")]'
)
splitter.filter_add('@ignored="false"')
if opts.filter_by:
for filter_by in opts.filter_by:
splitter.filter_add(filter_by)
if opts.group_by:
for group_by in opts.group_by:
splitter.group_by(group_by)
splitter.split()
result = splitter.propose_assignment(stagings)
if result is not True:
print('Failed to generate proposal: {}'.format(result))
return
proposal = splitter.proposal
if len(proposal) == 0:
print('Empty proposal')
return
if opts.interactive:
with tempfile.NamedTemporaryFile(suffix='.yml') as temp:
temp.write(
yaml.safe_dump(splitter.proposal,
default_flow_style=False) + '\n\n')
temp.write(
'# move requests between stagings or comment/remove them\n'
)
temp.write('# change the target staging for a group\n')
temp.write('# stagings\n')
temp.write('# - considered: {}\n'.format(', '.join(
sorted(splitter.stagings_considerable.keys()))))
temp.write('# - remaining: {}\n'.format(', '.join(
sorted(splitter.stagings_available.keys()))))
temp.flush()
editor = os.getenv('EDITOR')
if not editor:
editor = 'xdg-open'
return_code = subprocess.call([editor, temp.name])
proposal = yaml.safe_load(open(temp.name).read())
print(yaml.safe_dump(proposal, default_flow_style=False))
print('Accept proposal? [y/n] (y): ', end='')
response = raw_input().lower()
if response != '' and response != 'y':
print('Quit')
return
for group in sorted(proposal.keys()):
g = proposal[group]
if not g['requests']:
# Skipping since all request removed, presumably in interactive.
continue
print('Staging {}'.format(g['staging']))
# SelectCommand expects strings.
request_ids = map(str, g['requests'].keys())
target_project = api.prj_from_short(g['staging'])
SelectCommand(api, target_project) \
.perform(request_ids, opts.move, opts.from_, opts.no_freeze)
else:
target_project = api.prj_from_short(stagings[0])
if opts.add:
api.mark_additional_packages(target_project, [opts.add])
else:
SelectCommand(api, target_project) \
.perform(requests, opts.move, opts.from_, opts.no_freeze)
elif cmd == 'cleanup_rings':
CleanupRings(api).perform()
elif cmd == 'ignore':
IgnoreCommand(api).perform(args[1:], opts.message)
elif cmd == 'unignore':
UnignoreCommand(api).perform(args[1:], opts.cleanup)
elif cmd == 'list':
ListCommand(api).perform(args[1:], supersede=opts.supersede)
elif cmd == 'adi':
AdiCommand(api).perform(args[1:],
move=opts.move,
by_dp=opts.by_develproject,
split=opts.split)
elif cmd == 'repair':
RepairCommand(api).perform(args[1:])
