def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
project = self.api.project
curr_version = date.today().strftime('%Y%m%d')
update_version_attr = False
url = self.api.makeurl(['source', project], {'view': 'productlist'})
products = ET.parse(http_GET(url)).getroot()
for product in products.findall('product'):
product_name = product.get('name') + '.product'
product_pkg = product.get('originpackage')
url = self.api.makeurl(['source', project, product_pkg, product_name])
product_spec = http_GET(url).read()
new_product = re.sub(r'<version>\d{8}</version>', '<version>%s</version>' % curr_version, product_spec)
if product_spec != new_product:
update_version_attr = True
http_PUT(url + '?comment=Update+version', data=new_product)
if update_version_attr:
self.update_version_attribute(project, curr_version)
ports_prjs = ['PowerPC', 'ARM', 'zSystems' ]
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project) and update_version_attr:
self.update_version_attribute(project, curr_version)
def check_depinfo_ring(self, prj, nextprj):
url = makeurl(self.api.apiurl, ['build', prj, '_result'])
root = ET.parse(http_GET(url)).getroot()
for repo in root.findall('result'):
repostate = repo.get('state', 'missing')
if repostate not in ['unpublished', 'published'] or repo.get('dirty', 'false') == 'true':
print('Repo {}/{} is in state {}'.format(repo.get('project'), repo.get('repository'), repostate))
return False
for package in repo.findall('status'):
code = package.get('code')
if code not in ['succeeded', 'excluded', 'disabled']:
print('Package {}/{}/{} is {}'.format(repo.get('project'), repo.get('repository'), package.get('package'), code))
return False
self.find_inner_ring_links(prj)
for arch in self.api.cstaging_dvd_archs:
self.fill_pkgdeps(prj, 'standard', arch)
if prj == '{}:1-MinimalX'.format(self.api.crings):
url = makeurl(self.api.apiurl, ['build', prj, 'images', arch, 'Test-DVD-' + arch, '_buildinfo'])
root = ET.parse(http_GET(url)).getroot()
for bdep in root.findall('bdep'):
if 'name' not in bdep.attrib:
continue
b = bdep.attrib['name']
if b not in self.bin2src:
continue
b = self.bin2src[b]
self.pkgdeps[b] = 'MYdvd'
if prj == '{}:2-TestDVD'.format(self.api.crings):
url = makeurl(self.api.apiurl, ['build', prj, 'images', arch, 'Test-DVD-' + arch, '_buildinfo'])
root = ET.parse(http_GET(url)).getroot()
for bdep in root.findall('bdep'):
if 'name' not in bdep.attrib:
continue
b = bdep.attrib['name']
if b not in self.bin2src:
continue
b = self.bin2src[b]
self.pkgdeps[b] = 'MYdvd2'
if prj == '{}:0-Bootstrap'.format(self.api.crings):
url = makeurl(self.api.apiurl, ['build', prj, 'standard', '_buildconfig'])
for line in http_GET(url).read().split('\n'):
if line.startswith('Preinstall:') or line.startswith('Support:'):
for prein in line.split(':')[1].split():
if prein not in self.bin2src:
continue
b = self.bin2src[prein]
self.pkgdeps[b] = 'MYinstall'
for source in self.sources:
if source not in self.pkgdeps and source not in self.links:
if source.startswith('texlive-specs-'): # XXX: texlive bullshit packaging
continue
print('osc rdelete -m cleanup {} {}'.format(prj, source))
if nextprj:
print('osc linkpac {} {} {}').format(self.api.project, source, nextprj)
def entity_exists(apiurl, project, package=None):
try:
http_GET(makeurl(apiurl, filter(None, ['source', project, package]) + ['_meta']))
except HTTPError as e:
if e.code == 404:
return False
raise e
return True
def project_exists(self, project):
"""
Return true if the given project exists
:param project: project name to check
"""
url = self.makeurl(['source', project, '_meta'])
try:
http_GET(url)
except urllib2.HTTPError:
return False
return True
def do_viewrequest(self, subcmd, opts, *args):
"""${cmd_name}: view the raw content of a request
Usage:
${cmd_name} SR ...
View the raw content of a request
${cmd_option_list}
"""
apiurl = self.get_api_url()
for sr in args:
url = makeurl(apiurl, ['request', str(sr)])
print http_GET(url).read()
def item_exists(self, project, package=None):
"""
Return true if the given project exists
:param project: project name to check
:param package: optional package to check
"""
if package:
url = self.makeurl(['source', project, package, '_meta'])
else:
url = self.makeurl(['source', project, '_meta'])
try:
http_GET(url)
except urllib2.HTTPError:
return False
return True
def _checker_check_dups(self, project, opts):
url = makeurl(opts.apiurl, ['request'], "states=new,review&project=%s&view=collection" % project)
f = http_GET(url)
root = ET.parse(f).getroot()
rqs = {}
for rq in root.findall('request'):
id = rq.attrib['id']
for a in rq.findall('action'):
source = a.find('source')
target = a.find('target')
type = a.attrib['type']
assert target != None
if target.attrib['project'] != project:
continue
# print(id)
# ET.dump(target)
if 'package' not in target.attrib:
continue
package = target.attrib['package']
if type + package in rqs:
[oldid, oldsource] = rqs[type + package]
if oldid > id:
s = oldid
oldid = id
id = s
assert oldid < id
if source != None and oldsource != None:
if (source.attrib['project'] == oldsource.attrib['project'] and
source.attrib['package'] == oldsource.attrib['package']):
change_request_state(opts.apiurl, str(oldid), 'superseded',
'superseded by %s' % id, id)
continue
print("DUPS found:", id, oldid)
rqs[type + package] = [id, source]
def getCommitLog(self, project, package, revision=None):
"""
getCommitLog(project, package, revision=None) -> list
Get commit log for package in project. If revision is set, get just the
log for that revision.
Each log is a tuple of the form (rev, srcmd5, version, time, user,
comment)
"""
u = core.makeurl(self.apiurl, ['source', project, package, '_history'])
f = core.http_GET(u)
root = ElementTree.parse(f).getroot()
r = []
revisions = root.findall('revision')
revisions.reverse()
for node in revisions:
rev = int(node.get('rev'))
if revision and rev != int(revision):
continue
srcmd5 = node.find('srcmd5').text
version = node.find('version').text
user = node.find('user').text
try:
comment = node.find('comment').text
except:
comment = '<no message>'
t = time.localtime(int(node.find('time').text))
t = time.strftime('%Y-%m-%d %H:%M:%S', t)
r.append((rev, srcmd5, version, t, user, comment))
return r
def package_source_hash(apiurl, project, package, revision=None):
query = {}
if revision:
query['rev'] = revision
# Will not catch packages that previous had a link, but no longer do.
if package_source_link_copy(apiurl, project, package):
query['expand'] = 1
try:
url = makeurl(apiurl, ['source', project, package], query)
root = ETL.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 400 or e.code == 404:
# 400: revision not found, 404: package not found.
return None
raise e
if revision and root.find('error') is not None:
# OBS returns XML error instead of HTTP 404 if revision not found.
return None
from osclib.util import sha1_short
return sha1_short(root.xpath('entry[@name!="_link"]/@md5'))
def getWorkerStatus(self):
"""
getWorkerStatus() -> list of dicts
Get worker status as a list of dictionaries. Each dictionary contains the keys 'id',
'hostarch', and 'status'. If the worker is building, the dict will additionally contain the
keys 'project', 'package', 'target', and 'starttime'
"""
url = core.makeurl(self.apiurl, ['build', '_workerstatus'])
f = core.http_GET(url)
tree = ElementTree.parse(f).getroot()
workerstatus = []
for worker in tree.findall('building'):
d = {'id': worker.get('workerid'),
'status': 'building'}
for attr in ('hostarch', 'project', 'package', 'starttime'):
d[attr] = worker.get(attr)
d['target'] = '/'.join((worker.get('repository'), worker.get('arch')))
d['started'] = time.asctime(time.localtime(float(worker.get('starttime'))))
workerstatus.append(d)
for worker in tree.findall('idle'):
d = {'id': worker.get('workerid'),
'hostarch': worker.get('hostarch'),
'status': 'idle'}
workerstatus.append(d)
return workerstatus
def getSubmitRequests(self):
"""
getSubmitRequests() -> list of dicts
"""
url = core.makeurl(self.apiurl, ['search', 'request', '?match=submit'])
f = core.http_GET(url)
tree = ElementTree.parse(f).getroot()
submitrequests = []
for sr in tree.findall('request'):
if sr.get('type') != "submit":
continue
d = {'id': int(sr.get('id'))}
sb = sr.findall('submit')[0]
src = sb.findall('source')[0]
d['srcproject'] = src.get('project')
d['srcpackage'] = src.get('package')
dst = sb.findall('target')[0]
d['dstproject'] = dst.get('project')
d['dstpackage'] = dst.get('package')
d['state'] = sr.findall('state')[0].get('name')
submitrequests.append(d)
submitrequests.sort(key=lambda x: x['id'])
return submitrequests
def find_request_package(self, package):
"""
Look up the package by its name and return the SR#
:param package: name of the package
"""
query = 'states=new,review&project={}&view=collection&package={}'
query = query.format(self.api.project, urllib2.quote(package))
url = makeurl(self.api.apiurl, ['request'], query)
f = http_GET(url)
root = ET.parse(f).getroot()
requests = []
for sr in root.findall('request'):
# Check the target matches - OBS query is case insensitive, but OBS is not
rq_target = sr.find('action').find('target')
if package != rq_target.get('package') or self.api.project != rq_target.get('project'):
continue
request = sr.get('id')
state = sr.find('state').get('name')
self.srs[int(request)] = {'project': self.api.project, 'state': state}
requests.append(request)
if len(requests) > 1:
msg = 'There are multiple requests for package "{}": {}'
msg = msg.format(package, ', '.join(requests))
raise oscerr.WrongArgs(msg)
request = int(requests[0]) if requests else None
return request
def fill_pkgdeps(self, prj, repo, arch):
url = makeurl(self.api.apiurl, ['build', prj, repo, arch, '_builddepinfo'])
f = http_GET(url)
root = ET.parse(f).getroot()
for package in root.findall('package'):
source = package.find('source').text
if package.attrib['name'].startswith('preinstall'):
continue
self.sources.add(source)
for subpkg in package.findall('subpkg'):
subpkg = subpkg.text
if subpkg in self.bin2src:
if self.bin2src[subpkg] == source:
# different archs
continue
print('Binary {} is defined twice: {}/{}'.format(subpkg, prj, source))
self.bin2src[subpkg] = source
for package in root.findall('package'):
source = package.find('source').text
for pkg in package.findall('pkgdep'):
if pkg.text not in self.bin2src:
print('Package {} not found in place'.format(pkg.text))
continue
b = self.bin2src[pkg.text]
self.pkgdeps[b] = source
def issue_trackers(apiurl):
url = makeurl(apiurl, ['issue_trackers'])
root = ET.parse(http_GET(url)).getroot()
trackers = {}
for tracker in root.findall('issue-tracker'):
trackers[tracker.find('name').text] = tracker.find('label').text
return trackers
def package_kind(apiurl, project, package):
if package.startswith('00'):
return 'meta'
if ':' in package:
return 'multibuild_subpackage'
if package.startswith('patchinfo.'):
return 'patchinfo'
try:
url = makeurl(apiurl, ['source', project, package, '_meta'])
root = ETL.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 404:
return None
raise e
if root.find('releasename') is not None:
return 'maintenance_update'
if root.find('bcntsynctag') is not None:
return 'multispec_subpackage'
# Some multispec subpackages do not have bcntsynctag, so check link.
link = entity_source_link(apiurl, project, package)
if link is not None and link.get('cicount') == 'copy':
kind_target = package_kind(apiurl, project, link.get('package'))
if kind_target != 'maintenance_update':
# If a multispec subpackage was updated via a maintenance update the
# proper link information is lost and it will be considered source.
return 'multispec_subpackage'
return 'source'
def fill_pkgdeps(self, prj, repo, arch):
url = makeurl(self.api.apiurl, ['build', prj, repo, arch, '_builddepinfo'])
f = http_GET(url)
root = ET.parse(f).getroot()
for package in root.findall('package'):
source = package.find('source').text
if package.attrib['name'].startswith('preinstall'):
continue
self.sources.append(source)
for subpkg in package.findall('subpkg'):
subpkg = subpkg.text
if subpkg in self.bin2src:
print('Binary {} is defined twice: {}/{}'.format(subpkg, prj, source))
self.bin2src[subpkg] = source
for package in root.findall('package'):
source = package.find('source').text
for pkg in package.findall('pkgdep'):
if pkg.text not in self.bin2src:
if pkg.text.startswith('texlive-'):
for letter in range(ord('a'), ord('z') + 1):
self.pkgdeps['texlive-specs-' + chr(letter)] = 'texlive-specs-' + chr(letter)
else:
print('Package {} not found in place'.format(pkg.text))
continue
b = self.bin2src[pkg.text]
self.pkgdeps[b] = source
def _get_verifymd5(self, p, rev):
try:
url = makeurl(self.get_api_url(), ['source', p.sproject, p.spackage, '?view=info&rev=%s' % rev])
root = ET.parse(http_GET(url)).getroot()
except urllib2.HTTPError, e:
print 'ERROR in URL %s [%s]' % (url, e)
return []
def find_request_id(self, request_id):
"""
Look up the request by ID to verify if it is correct
:param request_id: ID of the added request
"""
if not _is_int(request_id):
return False
url = makeurl(self.api.apiurl, ['request', str(request_id)])
try:
f = http_GET(url)
except urllib2.HTTPError:
return None
root = ET.parse(f).getroot()
if root.get('id', None) != str(request_id):
return None
project = root.find('action').find('target').get('project')
if (project != self.api.project and not project.startswith(self.api.cstaging)):
msg = 'Request {} is not for {}, but for {}'
msg = msg.format(request_id, self.api.project, project)
raise oscerr.WrongArgs(msg)
self.srs[int(request_id)] = {'project': project}
return True
def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
# If thereis not product defined for this project, show the
# warning and return.
if not self.api.cproduct:
warnings.warn('There is not product defined in the configuration file.')
return
project = self.api.project
url = self.api.makeurl(['source', project, '_product', self.api.cproduct])
product = http_GET(url).read()
curr_version = date.today().strftime('%Y%m%d')
new_product = re.sub(r'<version>\d{8}</version>', '<version>%s</version>' % curr_version, product)
if product != new_product:
http_PUT(url + '?comment=Update+version', data=new_product)
service = {'cmd': 'runservice'}
ports_prjs = ['PowerPC', 'ARM' ]
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project):
baseurl = ['source', project, '_product']
url = self.api.makeurl(baseurl, query=service)
self.api.retried_POST(url)
def entity_clone(apiurl_source, apiurl_target, path, sanitize=None, clone=None, after=None):
if not hasattr(entity_clone, 'cloned'):
entity_clone.cloned = []
if path[0] == 'source' and not project_fence(path[1]):
# Skip projects outside of fence by marking as cloned.
if path not in entity_clone.cloned:
entity_clone.cloned.append(path)
if path in entity_clone.cloned:
print('skip {}'.format('/'.join(path)))
return
print('clone {}'.format('/'.join(path)))
entity_clone.cloned.append(path)
url = makeurl(apiurl_source, path)
entity = ET.parse(http_GET(url)).getroot()
if sanitize:
sanitize(entity)
if clone:
clone(apiurl_source, apiurl_target, entity)
url = makeurl(apiurl_target, path)
http_PUT(url, data=ET.tostring(entity))
if after:
after(apiurl_source, apiurl_target, entity)
def build(apiurl, project, repo, arch, package):
root = None
try:
url = makeurl(apiurl, ['build', project, repo, arch, package])
root = http_GET(url).read()
except urllib2.HTTPError, e:
print 'ERROR in URL %s [%s]' % (url, e)
def set_prj_pseudometa(self, project, meta):
"""
Sets project description to the YAML of the provided object
:param project: project to save into
:param meta: data to save
"""
# Get current metadata
url = make_meta_url('prj', project, self.apiurl)
root = ET.parse(http_GET(url)).getroot()
# Find description
description = root.find('description')
# Order the requests and replace it with yaml
meta['requests'] = sorted(meta['requests'], key=lambda x: x['id'])
description.text = yaml.dump(meta)
# Find title
title = root.find('title')
# Put something nice into title as well
new_title = []
for request in meta['requests']:
new_title.append(request['package'])
nt = ', '.join(sorted(new_title))
title.text = nt[:240]
# Write XML back
url = make_meta_url('prj', project, self.apiurl, force=True)
http_PUT(url, data=ET.tostring(root))
def issue_tracker_by_url(apiurl, tracker_url):
url = makeurl(apiurl, ['issue_trackers'])
root = ETL.parse(http_GET(url)).getroot()
if not tracker_url.endswith('/'):
# All trackers are formatted with trailing slash.
tracker_url += '/'
return next(iter(root.xpath('issue-tracker[url[text()="{}"]]'.format(tracker_url)) or []), None)
def get_package_list_from_repository(self, project, repository, arch, package):
url = makeurl(self.apiurl, ('build', project, repository, arch, package))
files = []
try:
binaries = ET.parse(http_GET(url)).getroot()
for binary in binaries.findall('binary'):
filename = binary.attrib['filename']
mtime = int(binary.attrib['mtime'])
result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', filename)
if not result:
if filename == 'rpmlint.log':
files.append((filename, '', '', mtime))
continue
pname = result.group(1)
if pname.endswith('-debuginfo') or pname.endswith('-debuginfo-32bit'):
continue
if pname.endswith('-debugsource'):
continue
if result.group(4) == 'src':
continue
files.append((filename, pname, result.group(4), mtime))
except urllib2.HTTPError:
pass
# print " - WARNING: Can't found list of packages (RPM) for %s in %s (%s, %s)" % (
# package, project, repository, arch)
return files
def get_package_information(self, project, pkgname, rev=None):
"""
Get the revision packagename and source project to copy from
based on content provided
:param project: the project we are having the package in
:param pkgname: name of the package we want to identify
:return dict ( project, package, revision, md5sum )
"""
package_info = {}
query = {
'rev': rev
}
if rev:
url = self.makeurl(['source', project, pkgname], query=query)
else:
url = self.makeurl(['source', project, pkgname])
content = http_GET(url)
root = ET.parse(content).getroot()
package_info['dir_srcmd5'] = root.attrib['srcmd5']
linkinfo = root.find('linkinfo')
package_info['srcmd5'] = linkinfo.attrib['srcmd5']
package_info['rev'] = linkinfo.attrib.get('rev', None)
package_info['project'] = linkinfo.attrib['project']
package_info['package'] = linkinfo.attrib['package']
return package_info
def get_open_requests(self):
"""
Get all requests with open review for staging project
that are not yet included in any staging project
:return list of pending open review requests
"""
requests = []
# xpath query, using the -m, -r, -s options
where = "@by_group='{}'+and+@state='new'".format(self.cstaging_group)
projects = [format(self.project)]
if self.cnonfree:
projects.append(self.cnonfree)
targets = ["target[@project='{}']".format(p) for p in projects]
query = "match=state/@name='review'+and+review[{}]+and+({})".format(
where, '+or+'.join(targets))
url = self.makeurl(['search', 'request'], query)
f = http_GET(url)
root = ET.parse(f).getroot()
for rq in root.findall('request'):
requests.append(rq)
return requests
def old_md5(self, src_project, tgt_project, src_package, rev):
"""Recollect old MD5 for a package."""
# XXX TODO - instead of fixing the limit, use endtime to make
# sure that we have the correct time frame.
limit = 20
query = {
'package': src_package,
# 'code': 'succeeded',
'limit': limit,
}
repositories = self.get_project_repos(src_project,
tgt_project,
src_package, rev)
srcmd5_list = []
for repository, archs in repositories:
for arch, status in archs:
if srcmd5_list:
break
if status not in ('succeeded', 'outdated'):
continue
url = makeurl(self.apiurl, ('build', src_project,
repository, arch,
'_jobhistory'),
query=query)
try:
root = ET.parse(http_GET(url)).getroot()
srcmd5_list = [e.get('srcmd5') for e in root.findall('jobhist')]
except urllib2.HTTPError, e:
print('ERROR in URL %s [%s]' % (url, e))
def find_request_project(self, source_project, newcand):
"""
Look up the source project by its name and return the SR#(s)
:param source_project: name of the source project
:param newcand: the review state of staging-group must be new
"""
query = 'states=new,review&project={}&view=collection'.format(self.api.project)
url = makeurl(self.api.apiurl, ['request'], query)
f = http_GET(url)
root = ET.parse(f).getroot()
ret = None
for sr in root.findall('request'):
# ensure staging tool don't picks the processed request again
if newcand:
staging_group_states = [review.get('state') for review in sr.findall('review') if review.get('by_group') == self.api.cstaging_group]
if 'new' not in staging_group_states:
continue
for act in sr.findall('action'):
src = act.find('source')
if src is not None and src.get('project') == source_project:
request = int(sr.attrib['id'])
state = sr.find('state').get('name')
self.srs[request] = {'project': self.api.project, 'state': state}
ret = True
return ret
def get_filelist_for_package(self, pkgname, project, extension=None):
"""
Get a list of files inside a package container
:param package: the base packagename to be linked to
:param project: Project to verify
:param extension: Limit the file list to files with this extension
"""
filelist = []
query = {
'extension': extension
}
if extension:
url = self.makeurl(['source', project, pkgname], query=query)
else:
url = self.makeurl(['source', project, pkgname])
try:
content = http_GET(url)
for entry in ET.parse(content).getroot().findall('entry'):
filelist.append(entry.attrib['name'])
except urllib2.HTTPError, err:
if err.code == 404:
# The package we were supposed to query does not exist
# we can pass this up and return the empty filelist
pass
def package_binary_list(apiurl, project, repository, arch, package=None, strip_multibuild=True, exclude_src_debug=False):
path = ['build', project, repository, arch]
if package:
path.append(package)
url = makeurl(apiurl, path, {'view': 'binaryversions'})
root = ET.parse(http_GET(url)).getroot()
package_binaries = []
binary_map = {} # last duplicate wins
for binary_list in root:
package = binary_list.get('package')
if strip_multibuild:
package = package.split(':', 1)[0]
for binary in binary_list:
filename = binary.get('name')
result = re.match(RPM_REGEX, filename)
if not result:
continue
binary = BinaryParsed(package, result.group('filename'),
result.group('name'), result.group('arch'))
if exclude_src_debug and binary_src_debug(binary):
continue
package_binaries.append(binary)
binary_map[result.group('filename')] = package
return package_binaries, binary_map
def update_and_solve_target(self,
apiurl,
target_project,
target_config,
main_repo,
opts,
skip_release=False):
print('[{}] {}/{}: update and solve'.format(opts.scope, opts.project,
main_repo))
group = target_config.get('pkglistgen-group', '000package-groups')
product = target_config.get('pkglistgen-product', '000product')
release = target_config.get('pkglistgen-release',
'000release-packages')
url = makeurl(apiurl, ['source', opts.project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.options.dry:
undelete_package(apiurl, opts.project, product, 'revive')
# TODO disable build.
print('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not opts.force:
root = ET.fromstringlist(
show_results_meta(apiurl,
opts.project,
product,
repository=[main_repo],
multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(
root.xpath('result[@state="dirty"]')):
print('{}/{} build in progress'.format(opts.project, product))
return
checkout_list = [group, product]
if not skip_release:
checkout_list.append(release)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.options.dry:
undelete_package(apiurl, opts.project, product, 'revive')
print(
'{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse.urlparse(apiurl).hostname
cache_dir = save_cache_path('opensuse-packagelists', host,
opts.project)
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
release_dir = os.path.join(cache_dir, release)
for package in checkout_list:
checkout_package(apiurl,
opts.project,
package,
expand_link=True,
prj_dir=cache_dir)
if not skip_release:
self.unlink_all_except(release_dir)
self.unlink_all_except(product_dir)
self.copy_directory_contents(
group_dir, product_dir,
['supportstatus.txt', 'groups.yml', 'package-groups.changes'])
self.change_extension(product_dir, '.spec.in', '.spec')
self.options.input_dir = group_dir
self.options.output_dir = product_dir
self.postoptparse()
print('-> do_update')
self.do_update('update', opts)
print('-> do_solve')
opts.ignore_unresolvable = bool(
target_config.get('pkglistgen-ignore-unresolvable'))
opts.ignore_recommended = bool(
target_config.get('pkglistgen-ignore-recommended'))
opts.include_suggested = bool(
target_config.get('pkglistgen-include-suggested'))
opts.locale = target_config.get('pkglistgen-local')
opts.locales_from = target_config.get('pkglistgen-locales-from')
self.do_solve('solve', opts)
delete_products = target_config.get('pkglistgen-delete-products',
'').split(' ')
self.unlink_list(product_dir, delete_products)
print('-> product service')
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
print(
subprocess.check_output(
[PRODUCT_SERVICE, product_file, product_dir,
opts.project]))
delete_kiwis = target_config.get(
'pkglistgen-delete-kiwis-{}'.format(opts.scope), '').split(' ')
self.unlink_list(product_dir, delete_kiwis)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
if skip_release:
self.unlink_list(None, spec_files)
else:
self.move_list(spec_files, release_dir)
self.multibuild_from_glob(product_dir, '*.kiwi')
self.build_stub(product_dir, 'kiwi')
self.commit_package(product_dir)
if not skip_release:
self.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')
self.commit_package(release_dir)
def project_list_prefix(apiurl, prefix):
"""Get a list of project with the same prefix."""
query = {'match': 'starts-with(@name, "{}")'.format(prefix)}
url = makeurl(apiurl, ['search', 'project', 'id'], query)
root = ETL.parse(http_GET(url)).getroot()
return root.xpath('project/@name')
def search(apiurl, path, xpath, query={}):
query['match'] = xpath
url = makeurl(apiurl, ['search', path], query)
return ETL.parse(http_GET(url)).getroot()
def check_depinfo_ring(self, prj, nextprj):
url = makeurl(self.api.apiurl, ['build', prj, '_result'])
root = ET.parse(http_GET(url)).getroot()
for repo in root.findall('result'):
repostate = repo.get('state', 'missing')
if repostate not in ['unpublished', 'published'] or repo.get(
'dirty', 'false') == 'true':
print('Repo {}/{} is in state {}'.format(
repo.get('project'), repo.get('repository'), repostate))
return False
for package in repo.findall('status'):
code = package.get('code')
if code not in ['succeeded', 'excluded', 'disabled']:
print('Package {}/{}/{} is {}'.format(
repo.get('project'), repo.get('repository'),
package.get('package'), code))
return False
self.find_inner_ring_links(prj)
for arch in ['x86_64', 'ppc64le']:
self.fill_pkgdeps(prj, 'standard', arch)
if prj == '{}:1-MinimalX'.format(self.api.crings):
url = makeurl(self.api.apiurl, [
'build', prj, 'images', arch, 'Test-DVD-' + arch,
'_buildinfo'
])
root = ET.parse(http_GET(url)).getroot()
for bdep in root.findall('bdep'):
if 'name' not in bdep.attrib:
continue
b = bdep.attrib['name']
if b not in self.bin2src:
continue
b = self.bin2src[b]
self.pkgdeps[b] = 'MYdvd'
if prj == '{}:2-TestDVD'.format(self.api.crings):
url = makeurl(self.api.apiurl, [
'build', prj, 'images', arch, 'Test-DVD-' + arch,
'_buildinfo'
])
root = ET.parse(http_GET(url)).getroot()
for bdep in root.findall('bdep'):
if 'name' not in bdep.attrib:
continue
b = bdep.attrib['name']
if b not in self.bin2src:
continue
b = self.bin2src[b]
self.pkgdeps[b] = 'MYdvd2'
if prj == '{}:0-Bootstrap'.format(self.api.crings):
url = makeurl(self.api.apiurl,
['build', prj, 'standard', '_buildconfig'])
for line in http_GET(url).read().split('\n'):
if line.startswith('Preinstall:') or line.startswith(
'Support:'):
for prein in line.split(':')[1].split():
if prein not in self.bin2src:
continue
b = self.bin2src[prein]
self.pkgdeps[b] = 'MYinstall'
for source in self.sources:
if source not in self.pkgdeps and source not in self.links:
print('osc rdelete -m cleanup {} {}'.format(prj, source))
if nextprj:
print('osc linkpac {} {} {}').format(
self.api.project, source, nextprj)
def package_source_changed(apiurl, project, package):
url = makeurl(apiurl, ['source', project, package, '_history'], {'limit': 1})
root = ETL.parse(http_GET(url)).getroot()
return datetime.fromtimestamp(int(root.find('revision/time').text), timezone.utc).replace(tzinfo=None)
def repository_arch_state(apiurl, project, repository, arch):
# just checking the mtimes of the repository's binaries
url = makeurl(apiurl, ['build', project, repository, arch, '_repository'])
from osclib.util import sha1_short
return sha1_short(http_GET(url).read())
def update_and_solve_target(self, api, target_project, target_config, main_repo,
project, scope, force, no_checkout,
only_release_packages, stop_after_solve):
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
self.repos = self.expand_repos(project, main_repo)
print('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
group = target_config.get('pkglistgen-group', '000package-groups')
product = target_config.get('pkglistgen-product', '000product')
release = target_config.get('pkglistgen-release', '000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
print('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
print('{}/{} build in progress'.format(project, product))
return
drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
print('{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
cache_dir = CacheManager.directory('pkglistgen', host, project)
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
release_dir = os.path.join(cache_dir, release)
oldrepos_dir = os.path.join(cache_dir, oldrepos)
for package in checkout_list:
if no_checkout:
print('Skipping checkout of {}/{}'.format(project, package))
continue
checkout_package(api.apiurl, project, package, expand_link=True, prj_dir=cache_dir)
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
file_utils.copy_directory_contents(group_dir, product_dir,
['supportstatus.txt', 'groups.yml',
'reference-unsorted.yml', 'reference-summary.yml',
'package-groups.changes'])
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
self.input_dir = group_dir
self.output_dir = product_dir
print('-> do_update')
# make sure we only calculcate existant architectures
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
self.update_repos(self.filtered_architectures)
if only_release_packages:
self.load_all_groups()
self.write_group_stubs()
else:
summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')),
locale = target_config.get('pkglistgen-locale'),
locales_from = target_config.get('pkglistgen-locales-from'))
if stop_after_solve:
return
if drop_list:
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
file_utils.unlink_list(product_dir, delete_products)
print('-> product service')
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
if not product_version:
# for stagings the product version doesn't matter (I hope)
product_version = '1'
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
self.replace_product_version(product_file, product_version)
print(subprocess.check_output(
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
file_utils.unlink_list(product_dir, delete_kiwis)
if scope == 'staging':
self.strip_medium_from_staging(product_dir)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
file_utils.move_list(spec_files, release_dir)
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
# filter special inc file
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
file_utils.move_list(inc_files, release_dir)
# do not overwrite weakremovers.inc if it exists
# we will commit there afterwards if needed
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
file_utils.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')
self.commit_package(release_dir)
if only_release_packages:
return
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
self.build_stub(product_dir, 'kiwi')
self.commit_package(product_dir)
error_output = b''
reference_summary = os.path.join(group_dir, 'reference-summary.yml')
if os.path.isfile(reference_summary):
summary_file = os.path.join(product_dir, 'summary.yml')
with open(summary_file, 'w') as f:
f.write('# Summary of packages in groups')
for group in sorted(summary):
# the unsorted group should appear filtered by
# unneeded.yml - so we need the content of unsorted.yml
# not unsorted.group (this grew a little unnaturally)
if group == 'unsorted':
continue
f.write('\n' + group + ':\n')
for package in sorted(summary[group]):
f.write(' - ' + package + '\n')
try:
error_output += subprocess.check_output(['diff', '-u', reference_summary, summary_file])
except subprocess.CalledProcessError as e:
error_output += e.output
reference_unsorted = os.path.join(group_dir, 'reference-unsorted.yml')
unsorted_file = os.path.join(product_dir, 'unsorted.yml')
try:
error_output += subprocess.check_output(['diff', '-u', reference_unsorted, unsorted_file])
except subprocess.CalledProcessError as e:
error_output += e.output
if len(error_output) > 0:
self.logger.error('Difference in yml:\n' + error_output.decode('utf-8'))
return True
def assertOBS(self):
url = makeurl(self.apiurl, ['about'])
root = ET.parse(http_GET(url)).getroot()
self.assertEqual(root.tag, 'about')
def _project_maintainer(self, request):
"""Get the list of maintainer of the target project."""
url = makeurl(self.apiurl, ('source', request.tgt_project, '_meta'))
root = ET.parse(http_GET(url)).getroot()
persons = [e.get('userid') for e in root.findall('.//person') if e.get('role') == 'maintainer']
return persons
parser.add_argument('-d', '--debug', action='store_true', default=False,
help='enable debug information')
args = parser.parse_args()
osc.conf.get_config(override_apiurl=args.apiurl)
osc.conf.config['debug'] = args.debug
apiurl = osc.conf.config['apiurl']
if apiurl.endswith('suse.de'):
amqp_prefix = 'suse'
openqa_url = 'https://openqa.suse.de'
else:
amqp_prefix = 'opensuse'
openqa_url = 'https://openqa.opensuse.org'
logging.basicConfig(level=logging.INFO)
listener = Listener(amqp_prefix, openqa_url)
url = makeurl(apiurl, ['search', 'project', 'id'], {'match': 'attribute/@name="OSRT:OpenQAMapping"'})
f = http_GET(url)
root = ET.parse(f).getroot()
for entry in root.findall('project'):
listener.add(Project(entry.get('name')))
try:
listener.run(runtime=10800)
except KeyboardInterrupt:
listener.stop()
def check_arch(self, architecture):
url = makeurl(self.apiurl,
['build', self.project, self.repository, architecture],
{'view': 'status'})
root = ET.parse(http_GET(url)).getroot()
return root.get('code') == 'finished'
'_jobhistory'),
query=query)
try:
root = ET.parse(http_GET(url)).getroot()
srcmd5_list = [e.get('srcmd5') for e in root.findall('jobhist')]
except urllib2.HTTPError, e:
print('ERROR in URL %s [%s]' % (url, e))
md5_set = set()
for srcmd5 in srcmd5_list:
query = {
'expand': 1,
'rev': srcmd5,
}
url = makeurl(self.apiurl, ('source', src_project, src_package), query=query)
root = ET.parse(http_GET(url)).getroot()
md5_set.add(root.find('linkinfo').get('srcmd5'))
return md5_set
def check_specs(self, request_id=None, request=None):
"""Check a single request and load the different SPECs files.
This method have side effects, it can ACCEPT or DECLINE
requests after some checks.
"""
requests = []
if request_id:
def fileinfo_ext(apiurl, project, repo, arch, package, filename):
url = makeurl(apiurl, ['build', project, repo, arch, package, filename],
{'view': 'fileinfo_ext'})
return ET.parse(http_GET(url)).getroot()
def get_packagelist(self, project, sle_pkglist=[], by_project=True):
"""
Return the list of package's info of a project.
If the latest package is from an incident then returns incident
package.
"""
pkglist = {}
packageinfo = {}
query = {'expand': 1}
root = ET.parse(
http_GET(makeurl(self.apiurl, ['source', project],
query=query))).getroot()
for i in root.findall('entry'):
pkgname = i.get('name')
orig_project = i.get('originproject')
is_incidentpkg = False
# Metapackage should not be selected
if pkgname.startswith('00') or\
pkgname.startswith('_') or\
pkgname.startswith('patchinfo.') or\
pkgname.startswith('skelcd-') or\
pkgname.startswith('installation-images') or\
pkgname.startswith('Leap-release') or\
pkgname.endswith('-mini') or\
'-mini.' in pkgname:
continue
# Ugly hack for package has dot in source package name
# eg. go1.x incidents as the name would be go1.x.xxx
if '.' in pkgname and re.match(r'[0-9]+$', pkgname.split('.')[-1]) and \
orig_project.startswith('SUSE:') and orig_project.endswith(':Update'):
is_incidentpkg = True
if pkgname.startswith('go1') or\
pkgname.startswith('bazel0') or\
pkgname.startswith('dotnet') or\
pkgname.startswith('rust1') or\
pkgname.startswith('ruby2'):
if not (pkgname.count('.') > 1):
is_incidentpkg = False
# If an incident found then update the package origin info
if is_incidentpkg:
orig_name = re.sub(r'\.[0-9]+$', '', pkgname)
incident_number = int(pkgname.split('.')[-1])
if orig_name in pkglist and pkglist[orig_name][
'Project'] == orig_project:
if re.match(r'[0-9]+$',
pkglist[orig_name]['Package'].split('.')[-1]):
old_incident_number = int(
pkglist[orig_name]['Package'].split('.')[-1])
if incident_number > old_incident_number:
pkglist[orig_name]['Package'] = pkgname
else:
pkglist[orig_name]['Package'] = pkgname
else:
pkglist[pkgname] = {
'Project': orig_project,
'Package': pkgname
}
if sle_pkglist and pkgname in sle_pkglist and not orig_project.startswith(
'openSUSE'):
pkglist[pkgname] = {
'Project': sle_pkglist[pkgname]['Project'],
'Package': sle_pkglist[pkgname]['Package']
}
if by_project:
for pkg in pkglist.keys():
if pkglist[pkg]['Project'].startswith(
'SUSE:') and self.is_sle_specific(pkg):
continue
if pkglist[pkg]['Project'] not in packageinfo:
packageinfo[pkglist[pkg]['Project']] = []
if pkglist[pkg]['Package'] not in packageinfo[pkglist[pkg]
['Project']]:
packageinfo[pkglist[pkg]['Project']].append(
pkglist[pkg]['Package'])
return packageinfo
return pkglist
def builddepinfo(apiurl, project, repo, arch, order = False):
query = {}
if order:
query['view'] = 'order'
url = makeurl(apiurl, ['build', project, repo, arch, '_builddepinfo'], query)
return ETL.parse(http_GET(url)).getroot()
def update_and_solve_target(self, api, target_project, target_config, main_repo,
project, scope, force, no_checkout,
only_release_packages, stop_after_solve):
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
self.use_newest_version = str2bool(target_config.get('pkglistgen-use-newest-version', 'False'))
self.repos = self.expand_repos(project, main_repo)
logging.debug('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
group = target_config.get('pkglistgen-group', '000package-groups')
product = target_config.get('pkglistgen-product', '000product')
release = target_config.get('pkglistgen-release', '000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
logging.info('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
logging.info('{}/{} build in progress'.format(project, product))
return
drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
logging.info('{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
cache_dir = CacheManager.directory('pkglistgen', host, project)
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
release_dir = os.path.join(cache_dir, release)
oldrepos_dir = os.path.join(cache_dir, oldrepos)
self.input_dir = group_dir
self.output_dir = product_dir
for package in checkout_list:
if no_checkout:
logging.debug('Skipping checkout of {}/{}'.format(project, package))
continue
checkout_package(api.apiurl, project, package, expand_link=True,
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
# print('RET', self.handle_package_diff(project, f"{group_dir}/summary-staging.txt", f"{product_dir}/summary-staging.txt"))
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes']
ignore_list += self.group_input_files()
file_utils.copy_directory_contents(group_dir, product_dir, ignore_list)
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
logging.debug('-> do_update')
# make sure we only calculcate existant architectures
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
self.update_repos(self.filtered_architectures)
if only_release_packages:
self.load_all_groups()
self.write_group_stubs()
else:
summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
ignore_recommended=str2bool(
target_config.get('pkglistgen-ignore-recommended')),
locale=target_config.get('pkglistgen-locale'),
locales_from=target_config.get('pkglistgen-locales-from'))
if stop_after_solve:
return
if drop_list and not only_release_packages:
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
try:
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
except MismatchedRepoException:
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
return
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
file_utils.unlink_list(product_dir, delete_products)
logging.debug('-> product service')
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
if not product_version:
# for stagings the product version doesn't matter (I hope)
product_version = '1'
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
self.replace_product_version(product_file, product_version)
logging.debug(subprocess.check_output(
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
file_utils.unlink_list(product_dir, delete_kiwis)
if scope == 'staging':
self.strip_medium_from_staging(product_dir)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
file_utils.move_list(spec_files, release_dir)
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
# filter special inc file
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
file_utils.move_list(inc_files, release_dir)
# do not overwrite weakremovers.inc if it exists
# we will commit there afterwards if needed
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
file_utils.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')
self.commit_package(release_dir)
if only_release_packages:
return
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
self.build_stub(product_dir, 'kiwi')
reference_summary = os.path.join(group_dir, f'summary-{scope}.txt')
if os.path.isfile(reference_summary):
summary_file = os.path.join(product_dir, f'summary-{scope}.txt')
output = []
for group in summary:
for package in sorted(summary[group]):
output.append(f'{package}:{group}')
with open(summary_file, 'w') as f:
for line in sorted(output):
f.write(line + '\n')
self.commit_package(product_dir)
if os.path.isfile(reference_summary):
return self.handle_package_diff(project, reference_summary, summary_file)