class CheckRepo(object):
def __init__(self, apiurl, project, readonly=False, force_clean=False, debug=False):
"""CheckRepo constructor."""
self.apiurl = apiurl
self.project = project
self.staging = StagingAPI(apiurl, self.project)
self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)
# grouped = { id: staging, }
self.grouped = {}
# groups = { staging: [ids,], }
self.groups = {}
self._staging()
self.readonly = readonly
self.debug_enable = debug
def debug(self, *args):
if not self.debug_enable:
return
print ' '.join([i if isinstance(i, basestring) else pformat(i) for i in args])
def _staging(self):
"""Preload the groups of related request associated by the same
staging project.
"""
for project in self.staging.get_staging_projects():
# Get all the requests identifier for the project
requests = self.staging.get_prj_pseudometa(project)['requests']
requests = [req['id'] for req in requests]
# Note: Originally we recover also the request returned by
# list_requests_in_prj(). I guest that if the staging
# project is working properly, this method do not add any
# new request to the list.
if requests:
self.groups[project] = requests
self.grouped.update({req: project for req in requests})
def get_request_state(self, request_id):
"""Return the current state of the request."""
state = None
url = makeurl(self.apiurl, ('request', str(request_id)))
try:
root = ET.parse(http_GET(url)).getroot()
state = root.find('state').get('name')
except urllib2.HTTPError, e:
print('ERROR in URL %s [%s]' % (url, e))
return state
class CheckRepo(object):
def __init__(self, apiurl, project, readonly=False, force_clean=False, debug=False):
"""CheckRepo constructor."""
self.apiurl = apiurl
self.project = project
self.staging = StagingAPI(apiurl, self.project)
self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)
# grouped = { id: staging, }
self.grouped = {}
# groups = { staging: [ids,], }
self.groups = {}
self._staging()
self.readonly = readonly
self.debug_enable = debug
def debug(self, *args):
if not self.debug_enable:
return
print ' '.join([i if isinstance(i, basestring) else pformat(i) for i in args])
def _staging(self):
"""Preload the groups of related request associated by the same
staging project.
"""
for project in self.staging.get_staging_projects():
# Get all the requests identifier for the project
requests = self.staging.get_prj_pseudometa(project)['requests']
requests = [req['id'] for req in requests]
# Note: Originally we recover also the request returned by
# list_requests_in_prj(). I guest that if the staging
# project is working properly, this method do not add any
# new request to the list.
if requests:
self.groups[project] = requests
self.grouped.update({req: project for req in requests})
def get_request_state(self, request_id):
"""Return the current state of the request."""
state = None
url = makeurl(self.apiurl, ('request', str(request_id)))
try:
root = ET.parse(http_GET(url)).getroot()
state = root.find('state').get('name')
except urllib2.HTTPError, e:
print('ERROR in URL %s [%s]' % (url, e))
return state
class StagingHelper(object):
def __init__(self, project):
self.project = project
self.apiurl = osc.conf.config['apiurl']
Config(self.project)
self.api = StagingAPI(self.apiurl, self.project)
def get_support_package_list(self, project, repository):
f = osc.core.get_buildconfig(self.apiurl, project, repository).splitlines()
pkg_list = []
for line in f:
if re.match('Preinstall', line) or re.match('VM[Ii]nstall', line) or re.match('Support', line):
content = line.split(':')
variables = [x.strip() for x in content[1].split(' ')]
for var in variables:
if var != '' and var not in pkg_list:
if var.startswith('!') and var[1:] in pkg_list:
pkg_list.remove(var[1:])
else:
pkg_list.append(var)
return pkg_list
def get_project_binarylist(self, project, repository, arch):
query = {'view': 'binarylist', 'repository': repository, 'arch': arch}
root = ET.parse(http_GET(makeurl(self.apiurl, ['build', project, '_result'],
query=query))).getroot()
return root
def process_project_binarylist(self, project, repository, arch):
prj_binarylist = self.get_project_binarylist(project, repository, arch)
files = {}
for package in prj_binarylist.findall('./result/binarylist'):
for binary in package.findall('binary'):
result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', binary.attrib['filename'])
if not result:
continue
bname = result.group(1)
if bname.endswith('-debuginfo') or bname.endswith('-debuginfo-32bit'):
continue
if bname.endswith('-debugsource'):
continue
if bname.startswith('::import::'):
continue
if result.group(4) == 'src':
continue
files[bname] = package.attrib['package']
return files
def check_multiple_specs(self, project, packages):
expanded_packages = []
for pkg in packages:
query = {'expand': 1}
url = makeurl(self.apiurl, ['source', project, pkg], query=query)
try:
root = ET.parse(http_GET(url)).getroot()
except urllib2.HTTPError as e:
if e.code == 404:
continue
raise
for en in root.findall('entry'):
if en.attrib['name'].endswith('.spec'):
expanded_packages.append(en.attrib['name'][:-5])
return expanded_packages
def crawl(self):
"""Main method"""
rebuild_data = self.api.dashboard_content_load('support_pkg_rebuild')
if rebuild_data is None:
print "There is no support_pkg_rebuild file!"
return
logging.info('Gathering support package list from %s' % self.project)
support_pkgs = self.get_support_package_list(self.project, 'standard')
files = self.process_project_binarylist(self.project, 'standard', 'x86_64')
staging_projects = ["%s:%s"%(self.api.cstaging, p) for p in self.api.get_staging_projects_short()]
cand_sources = defaultdict(list)
for stg in staging_projects:
prj_meta = self.api.get_prj_pseudometa(stg)
prj_staged_packages = [req['package'] for req in prj_meta['requests']]
prj_expanded_packages = self.check_multiple_specs(self.project, prj_staged_packages)
for pkg in support_pkgs:
if files.get(pkg) and files.get(pkg) in prj_expanded_packages:
if files.get(pkg) not in cand_sources[stg]:
cand_sources[stg].append(files.get(pkg))
root = ET.fromstring(rebuild_data)
logging.info('Checking rebuild data...')
for stg in root.findall('staging'):
rebuild = stg.find('rebuild').text
suppkg_list = stg.find('supportpkg').text
need_rebuild = False
suppkgs = []
if suppkg_list:
suppkgs = suppkg_list.split(',')
stgname = stg.get('name')
if len(cand_sources[stgname]) and rebuild == 'unknown':
need_rebuild = True
stg.find('rebuild').text = 'needed'
new_suppkg_list = ','.join(cand_sources[stgname])
stg.find('supportpkg').text = new_suppkg_list
elif len(cand_sources[stgname]) and rebuild != 'unknown':
for cand in cand_sources[stgname]:
if cand not in suppkgs:
need_rebuild = True
stg.find('rebuild').text = 'needed'
break
new_suppkg_list = ','.join(cand_sources[stgname])
stg.find('supportpkg').text = new_suppkg_list
elif not len(cand_sources[stgname]):
stg.find('rebuild').text = 'unneeded'
stg.find('supportpkg').text = ''
if stg.find('rebuild').text == 'needed':
need_rebuild = True
if need_rebuild and not self.api.is_repo_dirty(stgname, 'standard'):
logging.info('Rebuild %s' % stgname)
osc.core.rebuild(self.apiurl, stgname, None, None, None)
stg.find('rebuild').text = 'unneeded'
logging.info('Updating support pkg list...')
rebuild_data_updated = ET.tostring(root)
logging.debug(rebuild_data_updated)
if rebuild_data_updated != rebuild_data:
self.api.dashboard_content_save(
'support_pkg_rebuild', rebuild_data_updated, 'support package rebuild')
class TestApiCalls(unittest.TestCase):
"""
Tests for various api calls to ensure we return expected content
"""
def setUp(self):
"""
Initialize the configuration
"""
self.obs = OBS()
Config('openSUSE:Factory')
self.api = StagingAPI(APIURL, 'openSUSE:Factory')
def tearDown(self):
"""Clean internal cache"""
if hasattr(self.api, '_invalidate_all'):
self.api._invalidate_all()
def test_ring_packages(self):
"""
Validate the creation of the rings.
"""
# our content in the XML files
# test content for listonly ie. list command
ring_packages = {
'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-1': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-2': 'openSUSE:Factory:Rings:2-TestDVD',
'git': 'openSUSE:Factory:Rings:2-TestDVD',
'wine': 'openSUSE:Factory:Rings:1-MinimalX',
}
self.assertEqual(ring_packages, self.api.ring_packages_for_links)
# test content for real usage
ring_packages = {
'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-1': 'openSUSE:Factory:Rings:1-MinimalX',
'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-2': 'openSUSE:Factory:Rings:2-TestDVD',
'git': 'openSUSE:Factory:Rings:2-TestDVD',
'wine': 'openSUSE:Factory:Rings:1-MinimalX',
}
self.assertEqual(ring_packages, self.api.ring_packages)
@unittest.skip("no longer approving non-ring packages")
def test_dispatch_open_requests(self):
"""
Test dispatching and closure of non-ring packages
"""
# Get rid of open requests
self.api.dispatch_open_requests()
# Check that we tried to close it
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate'])
# Try it again
self.api.dispatch_open_requests()
# This time there should be nothing to close
self.assertEqual(httpretty.last_request().method, 'GET')
def test_pseudometa_get_prj(self):
"""
Test getting project metadata from YAML in project description
"""
# Try to get data from project that has no metadata
data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A')
# Should be empty, but contain structure to work with
self.assertEqual(data, {'requests': []})
# Add some sample data
rq = {'id': '123', 'package': 'test-package'}
data['requests'].append(rq)
# Save them and read them back
self.api.set_prj_pseudometa('openSUSE:Factory:Staging:A', data)
test_data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A')
# Verify that we got back the same data
self.assertEqual(data, test_data)
def test_list_projects(self):
"""
List projects and their content
"""
# Prepare expected results
data = []
for prj in self.obs.staging_project:
data.append('openSUSE:Factory:Staging:' + prj)
# Compare the results
self.assertEqual(data, self.api.get_staging_projects())
def test_open_requests(self):
"""
Test searching for open requests
"""
requests = []
# get the open requests
requests = self.api.get_open_requests()
# Compare the results, we only care now that we got 1 of them not the content
self.assertEqual(1, len(requests))
def test_get_package_information(self):
"""
Test if we get proper project, name and revision from the staging informations
"""
package_info = {
'dir_srcmd5': '751efeae52d6c99de48164088a33d855',
'project': 'home:Admin',
'rev': '7b98ac01b8071d63a402fa99dc79331c',
'srcmd5': '7b98ac01b8071d63a402fa99dc79331c',
'package': 'wine'
}
# Compare the results, we only care now that we got 2 of them not the content
self.assertEqual(
package_info,
self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine'))
def test_request_id_package_mapping(self):
"""
Test whether we can get correct id for sr in staging project
"""
prj = 'openSUSE:Factory:Staging:B'
# Get rq
num = self.api.get_request_id_for_package(prj, 'wine')
self.assertEqual(333, num)
# Get package name
self.assertEqual('wine', self.api.get_package_for_request_id(prj, num))
def test_rm_from_prj(self):
prj = 'openSUSE:Factory:Staging:B'
pkg = 'wine'
full_name = prj + '/' + pkg
# Verify package is there
self.assertTrue(full_name in self.obs.links)
# Get rq number
num = self.api.get_request_id_for_package(prj, pkg)
# Delete the package
self.api.rm_from_prj(prj, package='wine')
# Verify package is not there
self.assertTrue(full_name not in self.obs.links)
# RQ is gone
self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg))
self.assertEqual(None, self.api.get_package_for_request_id(prj, num))
# Verify that review is closed
self.assertEqual('accepted', self.obs.requests[str(num)]['review'])
self.assertEqual('new', self.obs.requests[str(num)]['request'])
def test_rm_from_prj_2(self):
# Try the same with request number
prj = 'openSUSE:Factory:Staging:B'
pkg = 'wine'
full_name = prj + '/' + pkg
# Get rq number
num = self.api.get_request_id_for_package(prj, pkg)
# Delete the package
self.api.rm_from_prj(prj, request_id=num)
# Verify package is not there
self.assertTrue(full_name not in self.obs.links)
# RQ is gone
self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg))
self.assertEqual(None, self.api.get_package_for_request_id(prj, num))
# Verify that review is closed
self.assertEqual('accepted', self.obs.requests[str(num)]['review'])
self.assertEqual('new', self.obs.requests[str(num)]['request'])
def test_add_sr(self):
prj = 'openSUSE:Factory:Staging:A'
rq = '123'
# Running it twice shouldn't change anything
for i in range(2):
# Add rq to the project
self.api.rq_to_prj(rq, prj)
# Verify that review is there
self.assertEqual('new', self.obs.requests[rq]['review'])
self.assertEqual('review', self.obs.requests[rq]['request'])
self.assertEqual(self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A'),
{'requests': [{'id': 123, 'package': 'gcc', 'author': 'Admin'}]})
def test_create_package_container(self):
"""Test if the uploaded _meta is correct."""
self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine')
self.assertEqual(httpretty.last_request().method, 'PUT')
self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/></package>')
self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta')
self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine', disable_build=True)
self.assertEqual(httpretty.last_request().method, 'PUT')
self.assertEqual(httpretty.last_request().body, '<package name="wine"><title /><description /><build><disable /></build></package>')
self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta')
def test_review_handling(self):
"""Test whether accepting/creating reviews behaves correctly."""
# Add review
self.api.add_review('123', by_project='openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview'])
# Try to readd, should do anything
self.api.add_review('123', by_project='openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'GET')
# Accept review
self.api.set_review('123', 'openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate'])
# Try to accept it again should do anything
self.api.set_review('123', 'openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'GET')
# But we should be able to reopen it
self.api.add_review('123', by_project='openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview'])
def test_prj_from_letter(self):
# Verify it works
self.assertEqual(self.api.prj_from_letter('openSUSE:Factory'), 'openSUSE:Factory')
self.assertEqual(self.api.prj_from_letter('A'), 'openSUSE:Factory:Staging:A')
def test_frozen_mtime(self):
"""Test frozen mtime."""
# Testing frozen mtime
self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:A') > 8)
self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:B') < 1)
# U == unfrozen
self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:U') > 1000)
def test_frozen_enough(self):
"""Test frozen enough."""
# Testing frozen mtime
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:B'), True)
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:A'), False)
# U == unfrozen
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:U'), False)
def test_move(self):
"""Test package movement."""
init_data = self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine')
self.api.move_between_project('openSUSE:Factory:Staging:B', 333, 'openSUSE:Factory:Staging:A')
test_data = self.api.get_package_information('openSUSE:Factory:Staging:A', 'wine')
self.assertEqual(init_data, test_data)
class TestApiCalls(unittest.TestCase):
"""
Tests for various api calls to ensure we return expected content
"""
def setUp(self):
"""
Initialize the configuration
"""
self.obs = OBS()
Config('openSUSE:Factory')
self.api = StagingAPI(APIURL, 'openSUSE:Factory')
def tearDown(self):
"""Clean internal cache"""
if hasattr(self.api, '_invalidate_all'):
self.api._invalidate_all()
def test_ring_packages(self):
"""
Validate the creation of the rings.
"""
# our content in the XML files
# test content for listonly ie. list command
ring_packages = {
'apparmor': 'openSUSE:Factory:Rings:1-MinimalX',
'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-1': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap',
'wine': 'openSUSE:Factory:Rings:1-MinimalX',
}
self.assertEqual(ring_packages, self.api.ring_packages_for_links)
# test content for real usage
ring_packages = {
'apparmor': 'openSUSE:Factory:Rings:1-MinimalX',
'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap',
'elem-ring-1': 'openSUSE:Factory:Rings:1-MinimalX',
'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap',
'wine': 'openSUSE:Factory:Rings:1-MinimalX',
}
self.assertEqual(ring_packages, self.api.ring_packages)
@unittest.skip("no longer approving non-ring packages")
def test_dispatch_open_requests(self):
"""
Test dispatching and closure of non-ring packages
"""
# Get rid of open requests
self.api.dispatch_open_requests()
# Check that we tried to close it
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate'])
# Try it again
self.api.dispatch_open_requests()
# This time there should be nothing to close
self.assertEqual(httpretty.last_request().method, 'GET')
def test_pseudometa_get_prj(self):
"""
Test getting project metadata from YAML in project description
"""
# Try to get data from project that has no metadata
data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A')
# Should be empty, but contain structure to work with
self.assertEqual(data, {'requests': []})
# Add some sample data
rq = {'id': '123', 'package': 'test-package'}
data['requests'].append(rq)
# Save them and read them back
self.api.set_prj_pseudometa('openSUSE:Factory:Staging:A', data)
test_data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A')
# Verify that we got back the same data
self.assertEqual(data, test_data)
def test_list_projects(self):
"""
List projects and their content
"""
# Prepare expected results
data = []
for prj in self.obs.staging_project:
data.append('openSUSE:Factory:Staging:' + prj)
# Compare the results
self.assertEqual(data, self.api.get_staging_projects())
def test_open_requests(self):
"""
Test searching for open requests
"""
requests = []
# get the open requests
requests = self.api.get_open_requests()
# Compare the results, we only care now that we got 1 of them not the content
self.assertEqual(1, len(requests))
def test_get_package_information(self):
"""
Test if we get proper project, name and revision from the staging informations
"""
package_info = {
'dir_srcmd5': '751efeae52d6c99de48164088a33d855',
'project': 'home:Admin',
'rev': '7b98ac01b8071d63a402fa99dc79331c',
'srcmd5': '7b98ac01b8071d63a402fa99dc79331c',
'package': 'wine'
}
# Compare the results, we only care now that we got 2 of them not the content
self.assertEqual(
package_info,
self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine'))
def test_request_id_package_mapping(self):
"""
Test whether we can get correct id for sr in staging project
"""
prj = 'openSUSE:Factory:Staging:B'
# Get rq
num = self.api.get_request_id_for_package(prj, 'wine')
self.assertEqual(333, num)
# Get package name
self.assertEqual('wine', self.api.get_package_for_request_id(prj, num))
def test_rm_from_prj(self):
prj = 'openSUSE:Factory:Staging:B'
pkg = 'wine'
full_name = prj + '/' + pkg
# Verify package is there
self.assertTrue(full_name in self.obs.links)
# Get rq number
num = self.api.get_request_id_for_package(prj, pkg)
# Delete the package
self.api.rm_from_prj(prj, package='wine')
# Verify package is not there
self.assertTrue(full_name not in self.obs.links)
# RQ is gone
self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg))
self.assertEqual(None, self.api.get_package_for_request_id(prj, num))
# Verify that review is closed
self.assertEqual('accepted', self.obs.requests[str(num)]['review'])
self.assertEqual('new', self.obs.requests[str(num)]['request'])
def test_rm_from_prj_2(self):
# Try the same with request number
prj = 'openSUSE:Factory:Staging:B'
pkg = 'wine'
full_name = prj + '/' + pkg
# Get rq number
num = self.api.get_request_id_for_package(prj, pkg)
# Delete the package
self.api.rm_from_prj(prj, request_id=num)
# Verify package is not there
self.assertTrue(full_name not in self.obs.links)
# RQ is gone
self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg))
self.assertEqual(None, self.api.get_package_for_request_id(prj, num))
# Verify that review is closed
self.assertEqual('accepted', self.obs.requests[str(num)]['review'])
self.assertEqual('new', self.obs.requests[str(num)]['request'])
def test_add_sr(self):
prj = 'openSUSE:Factory:Staging:A'
rq = '123'
# Running it twice shouldn't change anything
for i in range(2):
# Add rq to the project
self.api.rq_to_prj(rq, prj)
# Verify that review is there
self.assertEqual('new', self.obs.requests[rq]['review'])
self.assertEqual('review', self.obs.requests[rq]['request'])
self.assertEqual(self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A'),
{'requests': [{'id': 123, 'package': 'gcc', 'author': 'Admin', 'type': 'submit'}]})
def test_create_package_container(self):
"""Test if the uploaded _meta is correct."""
self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine')
self.assertEqual(httpretty.last_request().method, 'PUT')
self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/></package>')
self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta')
self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine', disable_build=True)
self.assertEqual(httpretty.last_request().method, 'PUT')
self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/><build><disable/></build></package>')
self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta')
def test_review_handling(self):
"""Test whether accepting/creating reviews behaves correctly."""
# Add review
self.api.add_review('123', by_project='openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview'])
# Try to readd, should do anything
self.api.add_review('123', by_project='openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'GET')
# Accept review
self.api.set_review('123', 'openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate'])
# Try to accept it again should do anything
self.api.set_review('123', 'openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'GET')
# But we should be able to reopen it
self.api.add_review('123', by_project='openSUSE:Factory:Staging:A')
self.assertEqual(httpretty.last_request().method, 'POST')
self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview'])
def test_prj_from_letter(self):
# Verify it works
self.assertEqual(self.api.prj_from_letter('openSUSE:Factory'), 'openSUSE:Factory')
self.assertEqual(self.api.prj_from_letter('A'), 'openSUSE:Factory:Staging:A')
def test_frozen_mtime(self):
"""Test frozen mtime."""
# Testing frozen mtime
self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:A') > 8)
self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:B') < 1)
# U == unfrozen
self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:U') > 1000)
def test_frozen_enough(self):
"""Test frozen enough."""
# Testing frozen mtime
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:B'), True)
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:A'), False)
# U == unfrozen
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:U'), False)
def test_move(self):
"""Test package movement."""
init_data = self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine')
self.api.move_between_project('openSUSE:Factory:Staging:B', 333, 'openSUSE:Factory:Staging:A')
test_data = self.api.get_package_information('openSUSE:Factory:Staging:A', 'wine')
self.assertEqual(init_data, test_data)
class TestApiCalls(unittest.TestCase):
"""
Tests for various api calls to ensure we return expected content
"""
def setUp(self):
"""
Initialize the configuration
"""
self.obs = OBS()
Config("openSUSE:Factory")
self.api = StagingAPI(APIURL, "openSUSE:Factory")
def tearDown(self):
"""Clean internal cache"""
self.api._invalidate_all()
def test_ring_packages(self):
"""
Validate the creation of the rings.
"""
# our content in the XML files
ring_packages = {
"elem-ring-0": "openSUSE:Factory:Rings:0-Bootstrap",
"elem-ring-1": "openSUSE:Factory:Rings:1-MinimalX",
"elem-ring-2": "openSUSE:Factory:Rings:2-TestDVD",
"git": "openSUSE:Factory:Rings:2-TestDVD",
"wine": "openSUSE:Factory:Rings:1-MinimalX",
}
self.assertEqual(ring_packages, self.api.ring_packages)
@unittest.skip("no longer approving non-ring packages")
def test_dispatch_open_requests(self):
"""
Test dispatching and closure of non-ring packages
"""
# Get rid of open requests
self.api.dispatch_open_requests()
# Check that we tried to close it
self.assertEqual(httpretty.last_request().method, "POST")
self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"changereviewstate"])
# Try it again
self.api.dispatch_open_requests()
# This time there should be nothing to close
self.assertEqual(httpretty.last_request().method, "GET")
def test_pseudometa_get_prj(self):
"""
Test getting project metadata from YAML in project description
"""
# Try to get data from project that has no metadata
data = self.api.get_prj_pseudometa("openSUSE:Factory:Staging:A")
# Should be empty, but contain structure to work with
self.assertEqual(data, {"requests": []})
# Add some sample data
rq = {"id": "123", "package": "test-package"}
data["requests"].append(rq)
# Save them and read them back
self.api.set_prj_pseudometa("openSUSE:Factory:Staging:A", data)
test_data = self.api.get_prj_pseudometa("openSUSE:Factory:Staging:A")
# Verify that we got back the same data
self.assertEqual(data, test_data)
def test_list_projects(self):
"""
List projects and their content
"""
# Prepare expected results
data = []
for prj in self.obs.staging_project:
data.append("openSUSE:Factory:Staging:" + prj)
# Compare the results
self.assertEqual(data, self.api.get_staging_projects())
def test_open_requests(self):
"""
Test searching for open requests
"""
requests = []
# get the open requests
requests = self.api.get_open_requests()
# Compare the results, we only care now that we got 1 of them not the content
self.assertEqual(1, len(requests))
def test_get_package_information(self):
"""
Test if we get proper project, name and revision from the staging informations
"""
package_info = {
"dir_srcmd5": "751efeae52d6c99de48164088a33d855",
"project": "home:Admin",
"rev": "7b98ac01b8071d63a402fa99dc79331c",
"srcmd5": "7b98ac01b8071d63a402fa99dc79331c",
"package": "wine",
}
# Compare the results, we only care now that we got 2 of them not the content
self.assertEqual(package_info, self.api.get_package_information("openSUSE:Factory:Staging:B", "wine"))
def test_request_id_package_mapping(self):
"""
Test whether we can get correct id for sr in staging project
"""
prj = "openSUSE:Factory:Staging:B"
# Get rq
num = self.api.get_request_id_for_package(prj, "wine")
self.assertEqual(333, num)
# Get package name
self.assertEqual("wine", self.api.get_package_for_request_id(prj, num))
def test_rm_from_prj(self):
prj = "openSUSE:Factory:Staging:B"
pkg = "wine"
full_name = prj + "/" + pkg
# Verify package is there
self.assertTrue(full_name in self.obs.links)
# Get rq number
num = self.api.get_request_id_for_package(prj, pkg)
# Delete the package
self.api.rm_from_prj(prj, package="wine")
# Verify package is not there
self.assertTrue(full_name not in self.obs.links)
# RQ is gone
self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg))
self.assertEqual(None, self.api.get_package_for_request_id(prj, num))
# Verify that review is closed
self.assertEqual("accepted", self.obs.requests[str(num)]["review"])
self.assertEqual("new", self.obs.requests[str(num)]["request"])
def test_rm_from_prj_2(self):
# Try the same with request number
prj = "openSUSE:Factory:Staging:B"
pkg = "wine"
full_name = prj + "/" + pkg
# Get rq number
num = self.api.get_request_id_for_package(prj, pkg)
# Delete the package
self.api.rm_from_prj(prj, request_id=num)
# Verify package is not there
self.assertTrue(full_name not in self.obs.links)
# RQ is gone
self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg))
self.assertEqual(None, self.api.get_package_for_request_id(prj, num))
# Verify that review is closed
self.assertEqual("accepted", self.obs.requests[str(num)]["review"])
self.assertEqual("new", self.obs.requests[str(num)]["request"])
def test_add_sr(self):
prj = "openSUSE:Factory:Staging:A"
rq = "123"
# Running it twice shouldn't change anything
for i in range(2):
# Add rq to the project
self.api.rq_to_prj(rq, prj)
# Verify that review is there
self.assertEqual("new", self.obs.requests[rq]["review"])
self.assertEqual("review", self.obs.requests[rq]["request"])
self.assertEqual(
self.api.get_prj_pseudometa("openSUSE:Factory:Staging:A"),
{"requests": [{"id": 123, "package": "gcc", "author": "Admin"}]},
)
def test_create_package_container(self):
"""Test if the uploaded _meta is correct."""
self.api.create_package_container("openSUSE:Factory:Staging:B", "wine")
self.assertEqual(httpretty.last_request().method, "PUT")
self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/></package>')
self.assertEqual(httpretty.last_request().path, "/source/openSUSE:Factory:Staging:B/wine/_meta")
self.api.create_package_container("openSUSE:Factory:Staging:B", "wine", disable_build=True)
self.assertEqual(httpretty.last_request().method, "PUT")
self.assertEqual(
httpretty.last_request().body,
'<package name="wine"><title /><description /><build><disable /></build></package>',
)
self.assertEqual(httpretty.last_request().path, "/source/openSUSE:Factory:Staging:B/wine/_meta")
def test_review_handling(self):
"""Test whether accepting/creating reviews behaves correctly."""
# Add review
self.api.add_review("123", by_project="openSUSE:Factory:Staging:A")
self.assertEqual(httpretty.last_request().method, "POST")
self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"addreview"])
# Try to readd, should do anything
self.api.add_review("123", by_project="openSUSE:Factory:Staging:A")
self.assertEqual(httpretty.last_request().method, "GET")
# Accept review
self.api.set_review("123", "openSUSE:Factory:Staging:A")
self.assertEqual(httpretty.last_request().method, "POST")
self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"changereviewstate"])
# Try to accept it again should do anything
self.api.set_review("123", "openSUSE:Factory:Staging:A")
self.assertEqual(httpretty.last_request().method, "GET")
# But we should be able to reopen it
self.api.add_review("123", by_project="openSUSE:Factory:Staging:A")
self.assertEqual(httpretty.last_request().method, "POST")
self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"addreview"])
def test_prj_from_letter(self):
# Verify it works
self.assertEqual(self.api.prj_from_letter("openSUSE:Factory"), "openSUSE:Factory")
self.assertEqual(self.api.prj_from_letter("A"), "openSUSE:Factory:Staging:A")
def test_frozen_mtime(self):
"""Test frozen mtime."""
# Testing frozen mtime
self.assertTrue(self.api.days_since_last_freeze("openSUSE:Factory:Staging:A") > 8)
self.assertTrue(self.api.days_since_last_freeze("openSUSE:Factory:Staging:B") < 1)
# U == unfrozen
self.assertTrue(self.api.days_since_last_freeze("openSUSE:Factory:Staging:U") > 1000)
def test_frozen_enough(self):
"""Test frozen enough."""
# Testing frozen mtime
self.assertEqual(self.api.prj_frozen_enough("openSUSE:Factory:Staging:B"), True)
self.assertEqual(self.api.prj_frozen_enough("openSUSE:Factory:Staging:A"), False)
# U == unfrozen
self.assertEqual(self.api.prj_frozen_enough("openSUSE:Factory:Staging:U"), False)
def test_move(self):
"""Test package movement."""
init_data = self.api.get_package_information("openSUSE:Factory:Staging:B", "wine")
self.api.move_between_project("openSUSE:Factory:Staging:B", 333, "openSUSE:Factory:Staging:A")
test_data = self.api.get_package_information("openSUSE:Factory:Staging:A", "wine")
self.assertEqual(init_data, test_data)
class StagingHelper(object):
def __init__(self, project):
self.project = project
self.apiurl = osc.conf.config['apiurl']
self.debug = osc.conf.config['debug']
Config(self.project)
self.api = StagingAPI(self.apiurl, self.project)
def load_rebuild_data(self, project, package, filename):
url = makeurl(
self.apiurl,
['source', project, package, '{}?expand=1'.format(filename)])
try:
return http_GET(url)
except urllib2.HTTPError:
return None
def save_rebuild_data(self, project, package, filename, content):
url = makeurl(self.apiurl, ['source', project, package, filename])
http_PUT(url + '?comment=support+package+rebuild+update', data=content)
def rebuild_project(self, project):
query = {'cmd': 'rebuild'}
url = makeurl(self.apiurl, ['build', project], query=query)
http_POST(url)
def get_source_packages(self, project):
"""Return the list of packages in a project."""
query = {'expand': 1}
root = ET.parse(
http_GET(makeurl(self.apiurl, ['source', project],
query=query))).getroot()
packages = [i.get('name') for i in root.findall('entry')]
return packages
def get_support_package_list(self, project, repository):
f = osc.core.get_buildconfig(self.apiurl, project,
repository).splitlines()
pkg_list = []
for line in f:
if re.match('Preinstall', line) or re.match('VMinstall', line):
content = line.split(':')
variables = [x.strip() for x in content[1].split(' ')]
for var in variables:
if var != '' and var not in pkg_list:
if var.startswith('!') and var[1:] in pkg_list:
pkg_list.remove(var[1:])
else:
pkg_list.append(var)
return pkg_list
def get_project_binarylist(self, project, repository, arch):
query = {'view': 'binarylist', 'repository': repository, 'arch': arch}
root = ET.parse(
http_GET(
makeurl(self.apiurl, ['build', project, '_result'],
query=query))).getroot()
return root
def get_package_buildinfo(self, project, repository, arch, package):
url = makeurl(
self.apiurl,
['build', project, repository, arch, package, '_buildinfo'])
root = ET.parse(http_GET(url)).getroot()
return root
def get_buildinfo_version(self, project, package):
buildinfo = self.get_package_buildinfo(project, 'standard', 'x86_64',
package)
versrel = buildinfo.find('versrel')
version = versrel.split('-')[0]
return version
def process_project_binarylist(self, project, repository, arch):
prj_binarylist = self.get_project_binarylist(project, repository, arch)
files = {}
for package in prj_binarylist.findall('./result/binarylist'):
for binary in package.findall('binary'):
result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm',
binary.attrib['filename'])
if not result:
continue
bname = result.group(1)
if bname.endswith('-debuginfo') or bname.endswith(
'-debuginfo-32bit'):
continue
if bname.endswith('-debugsource'):
continue
if bname.startswith('::import::'):
continue
if result.group(4) == 'src':
continue
files[bname] = package.attrib['package']
return files
def crawl(self):
"""Main method"""
rebuild_data = self.load_rebuild_data(self.project + ':Staging',
'dashboard',
'support_pkg_rebuild')
if rebuild_data is None:
print "There is no support_pkg_rebuild file!"
return
logging.info('Gathering support package list from %s' % self.project)
support_pkgs = self.get_support_package_list(self.project, 'standard')
files = self.process_project_binarylist(self.project, 'standard',
'x86_64')
staging_projects = [
"%s:%s" % (self.api.cstaging, p)
for p in self.api.get_staging_projects_short()
]
cand_sources = defaultdict(list)
for stg in staging_projects:
prj_meta = self.api.get_prj_pseudometa(stg)
prj_staged_packages = [
req['package'] for req in prj_meta['requests']
]
for pkg in support_pkgs:
if files.get(pkg) and files.get(pkg) in prj_staged_packages:
if files.get(pkg) not in cand_sources[stg]:
cand_sources[stg].append(files.get(pkg))
tree = ET.parse(rebuild_data)
root = tree.getroot()
logging.info('Checking rebuild data...')
update_suppkgs = False
for stg in root.findall('staging'):
rebuild = stg.find('rebuild').text
suppkg_list = stg.find('supportpkg').text
need_rebuild = False
suppkgs = []
if suppkg_list:
suppkgs = suppkg_list.split(',')
stgname = stg.get('name')
if len(cand_sources[stgname]) and rebuild == 'unknown':
need_rebuild = True
stg.find('rebuild').text = 'needed'
update_suppkgs = True
new_suppkg_list = ','.join(cand_sources[stgname])
stg.find('supportpkg').text = new_suppkg_list
elif len(cand_sources[stgname]) and rebuild != 'unknown':
if rebuild == 'unneeded':
for cand in cand_sources[stgname]:
if cand not in suppkgs:
need_rebuild = True
stg.find('rebuild').text = 'needed'
break
update_suppkgs = True
new_suppkg_list = ','.join(cand_sources[stgname])
stg.find('supportpkg').text = new_suppkg_list
elif not len(cand_sources[stgname]):
stg.find('rebuild').text = 'unneeded'
if need_rebuild and not self.api.is_repo_dirty(
stgname, 'standard'):
logging.info('Rebuild %s' % stgname)
self.rebuild_project(stgname)
stg.find('rebuild').text = 'unneeded'
if update_suppkgs:
logging.info('Updating support pkg list...')
logging.debug(ET.tostring(root))
self.save_rebuild_data(self.project + ':Staging', 'dashboard',
'support_pkg_rebuild', ET.tostring(root))
class StagingHelper(object):
def __init__(self, project):
self.project = project
self.apiurl = osc.conf.config['apiurl']
Config(self.apiurl, self.project)
self.api = StagingAPI(self.apiurl, self.project)
def get_support_package_list(self, project, repository):
f = osc.core.get_buildconfig(self.apiurl, project, repository).splitlines()
pkg_list = []
for line in f:
if re.match('Preinstall', line) or re.match('VM[Ii]nstall', line) or re.match('Support', line):
content = line.split(':')
variables = [x.strip() for x in content[1].split(' ')]
for var in variables:
if var != '' and var not in pkg_list:
if var.startswith('!') and var[1:] in pkg_list:
pkg_list.remove(var[1:])
else:
pkg_list.append(var)
return pkg_list
def get_project_binarylist(self, project, repository, arch):
query = {'view': 'binaryversions'}
root = ET.parse(http_GET(makeurl(self.apiurl, ['build', project, repository, arch],
query=query))).getroot()
return root
def process_project_binarylist(self, project, repository, arch):
prj_binarylist = self.get_project_binarylist(project, repository, arch)
files = {}
for package in prj_binarylist.findall('./binaryversionlist'):
for binary in package.findall('binary'):
result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', binary.attrib['name'])
if not result:
continue
bname = result.group(1)
if bname.endswith('-debuginfo') or bname.endswith('-debuginfo-32bit'):
continue
if bname.endswith('-debugsource'):
continue
if bname.startswith('::import::'):
continue
if result.group(4) == 'src':
continue
files[bname] = package.attrib['package'].split(':', 1)[0]
return files
def check_multiple_specs(self, project, packages):
expanded_packages = []
for pkg in packages:
query = {'expand': 1}
url = makeurl(self.apiurl, ['source', project, pkg], query=query)
try:
root = ET.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 404:
continue
raise
for en in root.findall('entry'):
if en.attrib['name'].endswith('.spec'):
expanded_packages.append(en.attrib['name'][:-5])
return expanded_packages
def crawl(self):
"""Main method"""
rebuild_data = self.api.pseudometa_file_load('support_pkg_rebuild')
if rebuild_data is None:
print("There is no support_pkg_rebuild file!")
return
logging.info('Gathering support package list from %s' % self.project)
support_pkgs = self.get_support_package_list(self.project, 'standard')
files = self.process_project_binarylist(self.project, 'standard', 'x86_64')
staging_projects = ["%s:%s"%(self.api.cstaging, p) for p in self.api.get_staging_projects_short()]
cand_sources = defaultdict(list)
for stg in staging_projects:
prj_meta = self.api.get_prj_pseudometa(stg)
prj_staged_packages = [req['package'] for req in prj_meta['requests']]
prj_expanded_packages = self.check_multiple_specs(self.project, prj_staged_packages)
for pkg in support_pkgs:
if files.get(pkg) and files.get(pkg) in prj_expanded_packages:
if files.get(pkg) not in cand_sources[stg]:
cand_sources[stg].append(files.get(pkg))
root = ET.fromstring(rebuild_data)
logging.info('Checking rebuild data...')
for stg in root.findall('staging'):
rebuild = stg.find('rebuild').text
suppkg_list = stg.find('supportpkg').text
need_rebuild = False
suppkgs = []
if suppkg_list:
suppkgs = suppkg_list.split(',')
stgname = stg.get('name')
if len(cand_sources[stgname]) and rebuild == 'unknown':
need_rebuild = True
stg.find('rebuild').text = 'needed'
new_suppkg_list = ','.join(cand_sources[stgname])
stg.find('supportpkg').text = new_suppkg_list
elif len(cand_sources[stgname]) and rebuild != 'unknown':
for cand in cand_sources[stgname]:
if cand not in suppkgs:
need_rebuild = True
stg.find('rebuild').text = 'needed'
break
new_suppkg_list = ','.join(cand_sources[stgname])
stg.find('supportpkg').text = new_suppkg_list
elif not len(cand_sources[stgname]):
stg.find('rebuild').text = 'unneeded'
stg.find('supportpkg').text = ''
if stg.find('rebuild').text == 'needed':
need_rebuild = True
if need_rebuild and not self.api.is_repo_dirty(stgname, 'standard'):
logging.info('Rebuild %s' % stgname)
osc.core.rebuild(self.apiurl, stgname, None, None, None)
stg.find('rebuild').text = 'unneeded'
logging.info('Updating support pkg list...')
rebuild_data_updated = ET.tostring(root)
logging.debug(rebuild_data_updated)
if rebuild_data_updated != rebuild_data:
self.api.pseudometa_file_save(
'support_pkg_rebuild', rebuild_data_updated, 'support package rebuild')
