def cleanup(self, project):
if not self.api.item_exists(project):
return False
pkglist = self.api.list_packages(project)
clean_list = set(pkglist) - set(self.api.cstaging_nocleanup)
for package in clean_list:
print "[cleanup] deleted %s/%s" % (project, package)
delete_package(self.api.apiurl, project, package, force=True, msg="autocleanup")
# wipe Test-DVD binaries and breaks kiwi build
if project.startswith('openSUSE:'):
for package in pkglist:
if package.startswith('Test-DVD-'):
# intend to break the kiwi file
arch = package.split('-')[-1]
fakepkgname = 'I-am-breaks-kiwi-build'
oldkiwifile = self.api.load_file_content(project, package, 'PRODUCT-'+arch+'.kiwi')
if oldkiwifile is not None:
newkiwifile = re.sub(r'<repopackage name="openSUSE-release"/>', '<repopackage name="%s"/>' % fakepkgname, oldkiwifile)
self.api.save_file_content(project, package, 'PRODUCT-' + arch + '.kiwi', newkiwifile)
# do wipe binary now
query = { 'cmd': 'wipe' }
query['package'] = package
query['repository'] = 'images'
url = self.api.makeurl(['build', project], query)
try:
http_POST(url)
except urllib2.HTTPError, err:
# failed to wipe isos but we can just continue
pass
def add_review(self, request_id, by_project=None, by_group=None, msg=None):
"""
Adds review by project to the request
:param request_id: request to add review to
:param project: project to assign review to
"""
req = get_request(self.apiurl, str(request_id))
if not req:
raise oscerr.WrongArgs('Request {} not found'.format(request_id))
for i in req.reviews:
if by_project and i.by_project == by_project and i.state == 'new':
return
if by_group and i.by_group == by_group and i.state == 'new':
return
# don't try to change reviews if the request is dead
if req.state.name not in ('new', 'review'):
return
query = {}
if by_project:
query['by_project'] = by_project
if not msg:
msg = 'Being evaluated by staging project "{}"'
msg = msg.format(by_project)
if by_group:
query['by_group'] = by_group
if not msg:
msg = 'Being evaluated by group "{}"'.format(by_group)
if not query:
raise oscerr.WrongArgs('We need a group or a project')
query['cmd'] = 'addreview'
url = self.makeurl(['request', str(request_id)], query)
http_POST(url, data=msg)
def update_staging_status(self, staging):
openqa_infos = dict()
for iso in self.staging_projects[staging]['isos']:
self.fetch_openqa_jobs(staging, iso, openqa_infos)
buildid = self.staging_projects[staging].get('id')
if not buildid:
self.logger.info("I don't know the build id of " + staging)
return
# all openQA jobs are created at the same URL
url = self.api.makeurl(['status_reports', 'published', staging, 'images', 'reports', buildid])
# make sure the names are unique
taken_names = dict()
for id in openqa_infos:
name = openqa_infos[id]['name']
if name in taken_names:
openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" + openqa_infos[id]['machine']
# the other id
oid = taken_names[name]
openqa_infos[oid]['name'] = openqa_infos[oid]['name'] + "@" + openqa_infos[oid]['machine']
if openqa_infos[id]['name'] == openqa_infos[oid]['name']:
raise Exception(f'Names of job #{id} and #{oid} collide')
taken_names[name] = id
for info in openqa_infos.values():
xml = self.openqa_check_xml(info['url'], info['state'], 'openqa:' + info['name'])
try:
http_POST(url, data=xml)
except HTTPError:
self.logger.error('failed to post status to ' + url)
def add_review(self, request_id, by_project=None, by_group=None, msg=None):
"""
Adds review by project to the request
:param request_id: request to add review to
:param project: project to assign review to
"""
req = get_request(self.apiurl, str(request_id))
if not req:
raise oscerr.WrongArgs('Request {} not found'.format(request_id))
for i in req.reviews:
if by_project and i.by_project == by_project and i.state == 'new':
return
if by_group and i.by_group == by_group and i.state == 'new':
return
# don't try to change reviews if the request is dead
if req.state.name not in ('new', 'review'):
return
query = {}
if by_project:
query['by_project'] = by_project
if not msg:
msg = 'Being evaluated by staging project "{}"'
msg = msg.format(by_project)
if by_group:
query['by_group'] = by_group
if not msg:
msg = 'Being evaluated by group "{}"'.format(by_group)
if not query:
raise oscerr.WrongArgs('We need a group or a project')
query['cmd'] = 'addreview'
url = self.makeurl(['request', str(request_id)], query)
http_POST(url, data=msg)
def fetch_openqa_jobs(self, staging, iso):
buildid = self.staging_projects[staging].get('id')
if not buildid:
self.logger.info("I don't know the build id of " + staging)
return
# all openQA jobs are created at the same URL
url = self.api.makeurl(['status_reports', 'published', staging, 'images', 'reports', buildid])
openqa = self.listener.jobs_for_iso(iso)
# collect job infos to pick names
openqa_infos = dict()
for job in openqa:
print(staging, iso, job['id'], job['state'], job['result'],
job['settings']['MACHINE'], job['settings']['TEST'])
openqa_infos[job['id']] = {'url': self.listener.test_url(job)}
openqa_infos[job['id']]['state'] = self.map_openqa_result(job)
openqa_infos[job['id']]['name'] = job['settings']['TEST']
openqa_infos[job['id']]['machine'] = job['settings']['MACHINE']
# make sure the names are unique
taken_names = dict()
for id in openqa_infos:
name = openqa_infos[id]['name']
if name in taken_names:
openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" + openqa_infos[id]['machine']
# the other id
id = taken_names[name]
openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" + openqa_infos[id]['machine']
taken_names[name] = id
for info in openqa_infos.values():
xml = self.openqa_check_xml(info['url'], info['state'], 'openqa:' + info['name'])
try:
http_POST(url, data=xml)
except HTTPError:
self.logger.error('failed to post status to ' + url)
def _write(self, signature):
url = makeurl(self.apiurl, ['source', self.lock, '_attribute'])
data = """
<attributes>
<attribute namespace='%s' name='LockedBy'>
<value>%s</value>
</attribute>
</attributes>""" % (self.ns, signature)
http_POST(url, data=data)
def _write(self, signature):
url = makeurl(self.apiurl, ['source', self.lock, '_attribute', '%s:LockedBy' % self.ns])
data = """
<attributes>
<attribute namespace='%s' name='LockedBy'>
<value>%s</value>
</attribute>
</attributes>""" % (self.ns, signature)
http_POST(url, data=data)
def create_and_wipe_package(self, project, package):
"""
Helper function for delete requests
"""
# create build disabled package
self.create_package_container(project, package, disable_build=True)
# now trigger wipebinaries to emulate a delete
url = self.makeurl(['build', project],
{'cmd': 'wipe', 'package': package})
http_POST(url)
def create_and_wipe_package(self, project, package):
"""
Helper function for delete requests
"""
# create build disabled package
self.create_package_container(project, package, disable_build=True)
# now trigger wipebinaries to emulate a delete
url = self.makeurl(['build', project],
{'cmd': 'wipe', 'package': package})
http_POST(url)
def attribute_value_save(apiurl, project, name, value, namespace='OSRT'):
root = ET.Element('attributes')
attribute = ET.SubElement(root, 'attribute')
attribute.set('namespace', namespace)
attribute.set('name', name)
ET.SubElement(attribute, 'value').text = value
# The OBS API of attributes is super strange, POST to update.
url = makeurl(apiurl, ['source', project, '_attribute'])
http_POST(url, data=ET.tostring(root))
def attribute_value_save(apiurl, project, name, value, namespace='OSRT'):
root = ET.Element('attributes')
attribute = ET.SubElement(root, 'attribute')
attribute.set('namespace', namespace)
attribute.set('name', name)
ET.SubElement(attribute, 'value').text = value
# The OBS API of attributes is super strange, POST to update.
url = makeurl(apiurl, ['source', project, '_attribute'])
http_POST(url, data=ET.tostring(root))
def rebuild_pkg(self, package, prj, arch, code=None):
query = {'cmd': 'rebuild', 'arch': arch}
if package:
query['package'] = package
pkg = query['package']
u = self.makeurl(['build', prj], query=query)
try:
print "tried to trigger rebuild for project '%s' package '%s'" % (
prj, pkg)
http_POST(u)
except:
print "could not trigger rebuild for project '%s' package '%s'" % (
prj, pkg)
def change_review_state(self, request_id, newstate, message=''):
"""Based on osc/osc/core.py. Fixed 'by_user'."""
query = {
'cmd': 'changereviewstate',
'newstate': newstate,
# XXX TODO - We force the user here, check if the user
# expressed in .oscrc (with the password stored) have
# rights to become this user.
'by_user': '******',
}
review_state = self.get_review_state(request_id)
if review_state == 'accepted' and newstate != 'accepted':
print ' - Avoid change state %s -> %s (%s)' % (review_state, newstate, message)
code = 404
url = makeurl(self.apiurl, ('request', str(request_id)), query=query)
if self.readonly:
print 'DRY RUN: POST %s' % url
return 200
try:
root = ET.parse(http_POST(url, data=message)).getroot()
code = root.attrib['code']
except urllib2.HTTPError, e:
print('ERROR in URL %s [%s]' % (url, e))
def rebuild_pkg(self, package, prj, arch, code=None):
query = {
'cmd': 'rebuild',
'arch': arch
}
if package:
query['package'] = package
pkg = query['package']
u = self.makeurl(['build', prj], query=query)
try:
print "tried to trigger rebuild for project '%s' package '%s'" % (prj, pkg)
http_POST(u)
except:
print "could not trigger rebuild for project '%s' package '%s'" % (prj, pkg)
def add_comment(self,
request_id=None,
project_name=None,
package_name=None,
comment=None,
parent_id=None):
"""Add a comment in an object in OBS.
:param request_id: Request where to write a comment.
:param project_name: Project name where to write a comment.
:param package_name: Package name where to write a comment.
:param comment: Comment to be published.
:return: Comment id.
"""
if not comment:
raise ValueError('Empty comment.')
comment = self.truncate(comment.strip())
# OBS returns unicode from some APIs, but comment API does not accept
# when included. Rather than handle everywhere just strip here.
comment = comment.encode('ascii', 'ignore')
query = {}
if parent_id:
query['parent_id'] = parent_id
url = self._prepare_url(request_id, project_name, package_name, query)
return http_POST(url, data=comment)
def _check_repo_change_review_state(self,
opts,
id_,
newstate,
message='',
supersed=None):
"""Taken from osc/osc/core.py, improved:
- verbose option added,
- empty by_user=& removed.
- numeric id can be int().
"""
query = {
'cmd': 'changereviewstate',
'newstate': newstate,
'by_user': '******',
}
if supersed:
query['superseded_by'] = supersed
# if message:
# query['comment'] = message
code = 404
u = makeurl(opts.apiurl, ['request', str(id_)], query=query)
try:
f = http_POST(u, data=message)
root = ET.parse(f).getroot()
code = root.attrib['code']
except urllib2.HTTPError, e:
print 'ERROR in URL %s [%s]' % (u, e)
def change_review_state(self, request_id, newstate, message=''):
"""Based on osc/osc/core.py. Fixed 'by_user'."""
query = {
'cmd': 'changereviewstate',
'newstate': newstate,
# XXX TODO - We force the user here, check if the user
# expressed in .oscrc (with the password stored) have
# rights to become this user.
'by_user': '******',
}
review_state = self.get_review_state(request_id)
if review_state == 'accepted' and newstate != 'accepted':
print ' - Avoid change state %s -> %s (%s)' % (review_state,
newstate, message)
code = 404
url = makeurl(self.apiurl, ('request', str(request_id)), query=query)
if self.readonly:
print 'DRY RUN: POST %s' % url
return 200
try:
root = ET.parse(http_POST(url, data=message)).getroot()
code = root.attrib['code']
except urllib2.HTTPError, e:
print('ERROR in URL %s [%s]' % (url, e))
def cleanup(self, project):
if not self.api.item_exists(project):
return False
pkglist = self.api.list_packages(project)
clean_list = set(pkglist) - set(self.api.cnocleanup_packages)
for package in clean_list:
print("[cleanup] deleted %s/%s" % (project, package))
delete_package(self.api.apiurl,
project,
package,
force=True,
msg="autocleanup")
# wipe Test-DVD binaries and breaks kiwi build
if project.startswith('openSUSE:'):
for package in pkglist:
if package.startswith('Test-DVD-'):
# intend to break the kiwi file
arch = package.split('-')[-1]
fakepkgname = 'I-am-breaks-kiwi-build'
oldkiwifile = source_file_load(self.api.apiurl, project,
package,
'PRODUCT-' + arch + '.kiwi')
if oldkiwifile is not None:
newkiwifile = re.sub(
r'<repopackage name="openSUSE-release"/>',
'<repopackage name="%s"/>' % fakepkgname,
oldkiwifile)
source_file_save(self.api.apiurl, project, package,
'PRODUCT-' + arch + '.kiwi',
newkiwifile)
# do wipe binary now
query = {'cmd': 'wipe'}
query['package'] = package
query['repository'] = 'images'
url = self.api.makeurl(['build', project], query)
try:
http_POST(url)
except HTTPError as err:
# failed to wipe isos but we can just continue
pass
return True
def retried_POST(self, url):
try:
return http_POST(url)
except urllib2.HTTPError, e:
if e.code == 504:
print 'Timeout on {}'.format(url)
return '<status code="timeout"/>'
if e.code / 100 == 5:
print 'Retrying {}'.format(url)
return self.retried_POST(url)
raise e
def retried_POST(self, url):
try:
return http_POST(url)
except urllib2.HTTPError, e:
if e.code == 504:
print 'Timeout on {}'.format(url)
return '<status code="timeout"/>'
if e.code / 100 == 5:
print 'Retrying {}'.format(url)
return self.retried_POST(url)
raise e
def add_comment(self, request_id=None, project_name=None,
package_name=None, comment=None):
"""Add a comment in an object in OBS.
:param request_id: Request where to write a comment.
:param project_name: Project name where to write a comment.
:param package_name: Package name where to write a comment.
:param comment: Comment to be published.
:return: Comment id.
"""
if not comment:
raise ValueError('Empty comment.')
url = self._prepare_url(request_id, project_name, package_name)
return http_POST(url, data=comment)
def reqReview(self, reqid, user='', group='', msg=''):
""" This method is called to add review msg to a request
Success: return None
Failed: return string of error message
"""
try:
query = { 'cmd': 'addreview' }
if user:
query['by_user'] = user
if group:
query['by_group'] = group
u = core.makeurl(self.apiurl, ['request', reqid], query=query)
f = core.http_POST(u, data=msg)
root = ElementTree.parse(f).getroot()
root.get('code')
except Exception, e:
return str(e)
def add_comment(self,
request_id=None,
project_name=None,
package_name=None,
comment=None):
"""Add a comment in an object in OBS.
:param request_id: Request where to write a comment.
:param project_name: Project name where to write a comment.
:param package_name: Package name where to write a comment.
:param comment: Comment to be published.
:return: Comment id.
"""
if not comment:
raise ValueError('Empty comment.')
url = self._prepare_url(request_id, project_name, package_name)
return http_POST(url, data=comment)
def get_sub_packages(self, pkg, project=None):
"""
Returns a list of packages that need to be linked into rings
too. A package is actually a tuple of project and package name
"""
ret = []
if not project:
project = self.ring_packages.get(pkg)
if not project:
return ret
url = self.makeurl(['source', project, pkg], {'cmd': 'showlinked'})
# showlinked is a POST for rather bizzare reasons
f = http_POST(url)
root = ET.parse(f).getroot()
for pkg in root.findall('package'):
ret.append((pkg.get('project'), pkg.get('name')))
return ret
def get_sub_packages(self, pkg, project=None):
"""
Returns a list of packages that need to be linked into rings
too. A package is actually a tuple of project and package name
"""
ret = []
if not project:
project = self.ring_packages.get(pkg)
if not project:
return ret
url = self.makeurl(['source', project, pkg],
{'cmd': 'showlinked'})
# showlinked is a POST for rather bizzare reasons
f = http_POST(url)
root = ET.parse(f).getroot()
for pkg in root.findall('package'):
ret.append((pkg.get('project'), pkg.get('name')))
return ret
def add_comment(self, request_id=None, project_name=None,
package_name=None, comment=None, parent_id=None):
"""Add a comment in an object in OBS.
:param request_id: Request where to write a comment.
:param project_name: Project name where to write a comment.
:param package_name: Package name where to write a comment.
:param comment: Comment to be published.
:return: Comment id.
"""
if not comment:
raise ValueError('Empty comment.')
comment = self.truncate(comment.strip())
# OBS returns unicode from some APIs, but comment API does not accept
# when included. Rather than handle everywhere just strip here.
comment = comment.encode('ascii', 'ignore')
query = {}
if parent_id:
query['parent_id'] = parent_id
url = self._prepare_url(request_id, project_name, package_name, query)
return http_POST(url, data=comment)
def _check_repo_change_review_state(self, opts, id_, newstate, message='', supersed=None):
"""Taken from osc/osc/core.py, improved:
- verbose option added,
- empty by_user=& removed.
- numeric id can be int().
"""
query = {
'cmd': 'changereviewstate',
'newstate': newstate,
'by_user': '******',
}
if supersed:
query['superseded_by'] = supersed
# if message:
# query['comment'] = message
code = 404
url = makeurl(opts.apiurl, ['request', str(id_)], query=query)
try:
f = http_POST(url, data=message)
root = ET.parse(f).getroot()
code = root.attrib['code']
except urllib2.HTTPError, e:
print 'ERROR in URL %s [%s]' % (url, e)
def request_state_change(apiurl, request_id, state):
query = { 'newstate': state, 'cmd': 'changestate' }
url = makeurl(apiurl, ['request', request_id], query)
return ETL.parse(http_POST(url)).getroot().get('code')
def accept_all(self, projects, force=False, cleanup=True):
accept_all_green = len(projects) == 0
if accept_all_green:
print('Accepting all acceptable projects')
if force:
print('ERROR: Not compatible with force option')
return False
self.requests = { 'delete': [], 'submit': [] }
staging_packages = {}
if accept_all_green:
projects = self.api.get_staging_projects()
for prj in projects:
project = self.api.prj_from_letter(prj)
status = self.api.project_status(project)
if status.get('state') != 'acceptable':
if accept_all_green:
continue
if not force:
print('The project "{}" is not yet acceptable.'.format(project))
return False
staging_packages[project] = []
for request in status.findall('staged_requests/request'):
self.requests[request.get('type')].append(request.get('package'))
staging_packages[project].append(request.get('package'))
other_new = self.find_new_requests(self.api.project)
for req in other_new:
self.requests[req['type']].append(req['package'])
print('delete links to packages pending deletion...')
self.delete_linked()
# we have checked ourselves and accepting one staging project creates a race
# for the other staging projects to appear building again
opts = { 'force': '1' }
print('triggering staging accepts...')
for project in staging_packages.keys():
u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept'], opts)
http_POST(u)
for req in other_new:
print(f"Accepting request {req['id']}: {req['package']}")
change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project)
for project in sorted(staging_packages.keys()):
print(f'waiting for staging project {project} to be accepted')
while True:
status = self.api.project_status(project, reload=True)
if status.get('state') == 'empty':
break
print('{} requests still staged - waiting'.format(status.find('staged_requests').get('count')))
time.sleep(1)
self.api.accept_status_comment(project, staging_packages[project])
if self.api.is_adi_project(project):
self.api.delete_empty_adi_project(project)
continue
self.api.staging_deactivate(project)
self.reset_rebuild_data(project)
if cleanup:
self.cleanup(project)
for package in self.requests['submit']:
self.fix_linking_packages(package)
if self.api.project.startswith('openSUSE:'):
self.update_factory_version()
if self.api.crebuild and self.api.item_exists(self.api.crebuild):
self.sync_buildfailures()
return True
def person_clone_after(apiurl_source, apiurl_target, person):
url = makeurl(apiurl_target, ['person', person.find('login').text], {'cmd': 'change_password'})
http_POST(url, data='opensuse')
def person_clone_after(apiurl_source, apiurl_target, person):
url = makeurl(apiurl_target, ['person', person.find('login').text], {'cmd': 'change_password'})
http_POST(url, data='opensuse')