class CheckRepo(object): def __init__(self, apiurl, project, readonly=False, force_clean=False, debug=False): """CheckRepo constructor.""" self.apiurl = apiurl self.project = project self.staging = StagingAPI(apiurl, self.project) self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean) # grouped = { id: staging, } self.grouped = {} # groups = { staging: [ids,], } self.groups = {} self._staging() self.readonly = readonly self.debug_enable = debug def debug(self, *args): if not self.debug_enable: return print ' '.join([i if isinstance(i, basestring) else pformat(i) for i in args]) def _staging(self): """Preload the groups of related request associated by the same staging project. """ for project in self.staging.get_staging_projects(): # Get all the requests identifier for the project requests = self.staging.get_prj_pseudometa(project)['requests'] requests = [req['id'] for req in requests] # Note: Originally we recover also the request returned by # list_requests_in_prj(). I guest that if the staging # project is working properly, this method do not add any # new request to the list. if requests: self.groups[project] = requests self.grouped.update({req: project for req in requests}) def get_request_state(self, request_id): """Return the current state of the request.""" state = None url = makeurl(self.apiurl, ('request', str(request_id))) try: root = ET.parse(http_GET(url)).getroot() state = root.find('state').get('name') except urllib2.HTTPError, e: print('ERROR in URL %s [%s]' % (url, e)) return state
class StagingHelper(object): def __init__(self, project): self.project = project self.apiurl = osc.conf.config['apiurl'] Config(self.project) self.api = StagingAPI(self.apiurl, self.project) def get_support_package_list(self, project, repository): f = osc.core.get_buildconfig(self.apiurl, project, repository).splitlines() pkg_list = [] for line in f: if re.match('Preinstall', line) or re.match('VM[Ii]nstall', line) or re.match('Support', line): content = line.split(':') variables = [x.strip() for x in content[1].split(' ')] for var in variables: if var != '' and var not in pkg_list: if var.startswith('!') and var[1:] in pkg_list: pkg_list.remove(var[1:]) else: pkg_list.append(var) return pkg_list def get_project_binarylist(self, project, repository, arch): query = {'view': 'binarylist', 'repository': repository, 'arch': arch} root = ET.parse(http_GET(makeurl(self.apiurl, ['build', project, '_result'], query=query))).getroot() return root def process_project_binarylist(self, project, repository, arch): prj_binarylist = self.get_project_binarylist(project, repository, arch) files = {} for package in prj_binarylist.findall('./result/binarylist'): for binary in package.findall('binary'): result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', binary.attrib['filename']) if not result: continue bname = result.group(1) if bname.endswith('-debuginfo') or bname.endswith('-debuginfo-32bit'): continue if bname.endswith('-debugsource'): continue if bname.startswith('::import::'): continue if result.group(4) == 'src': continue files[bname] = package.attrib['package'] return files def check_multiple_specs(self, project, packages): expanded_packages = [] for pkg in packages: query = {'expand': 1} url = makeurl(self.apiurl, ['source', project, pkg], query=query) try: root = ET.parse(http_GET(url)).getroot() except urllib2.HTTPError as e: if e.code == 404: continue raise for en in root.findall('entry'): if en.attrib['name'].endswith('.spec'): expanded_packages.append(en.attrib['name'][:-5]) return expanded_packages def crawl(self): """Main method""" rebuild_data = self.api.dashboard_content_load('support_pkg_rebuild') if rebuild_data is None: print "There is no support_pkg_rebuild file!" return logging.info('Gathering support package list from %s' % self.project) support_pkgs = self.get_support_package_list(self.project, 'standard') files = self.process_project_binarylist(self.project, 'standard', 'x86_64') staging_projects = ["%s:%s"%(self.api.cstaging, p) for p in self.api.get_staging_projects_short()] cand_sources = defaultdict(list) for stg in staging_projects: prj_meta = self.api.get_prj_pseudometa(stg) prj_staged_packages = [req['package'] for req in prj_meta['requests']] prj_expanded_packages = self.check_multiple_specs(self.project, prj_staged_packages) for pkg in support_pkgs: if files.get(pkg) and files.get(pkg) in prj_expanded_packages: if files.get(pkg) not in cand_sources[stg]: cand_sources[stg].append(files.get(pkg)) root = ET.fromstring(rebuild_data) logging.info('Checking rebuild data...') for stg in root.findall('staging'): rebuild = stg.find('rebuild').text suppkg_list = stg.find('supportpkg').text need_rebuild = False suppkgs = [] if suppkg_list: suppkgs = suppkg_list.split(',') stgname = stg.get('name') if len(cand_sources[stgname]) and rebuild == 'unknown': need_rebuild = True stg.find('rebuild').text = 'needed' new_suppkg_list = ','.join(cand_sources[stgname]) stg.find('supportpkg').text = new_suppkg_list elif len(cand_sources[stgname]) and rebuild != 'unknown': for cand in cand_sources[stgname]: if cand not in suppkgs: need_rebuild = True stg.find('rebuild').text = 'needed' break new_suppkg_list = ','.join(cand_sources[stgname]) stg.find('supportpkg').text = new_suppkg_list elif not len(cand_sources[stgname]): stg.find('rebuild').text = 'unneeded' stg.find('supportpkg').text = '' if stg.find('rebuild').text == 'needed': need_rebuild = True if need_rebuild and not self.api.is_repo_dirty(stgname, 'standard'): logging.info('Rebuild %s' % stgname) osc.core.rebuild(self.apiurl, stgname, None, None, None) stg.find('rebuild').text = 'unneeded' logging.info('Updating support pkg list...') rebuild_data_updated = ET.tostring(root) logging.debug(rebuild_data_updated) if rebuild_data_updated != rebuild_data: self.api.dashboard_content_save( 'support_pkg_rebuild', rebuild_data_updated, 'support package rebuild')
class TestApiCalls(unittest.TestCase): """ Tests for various api calls to ensure we return expected content """ def setUp(self): """ Initialize the configuration """ self.obs = OBS() Config('openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') def tearDown(self): """Clean internal cache""" if hasattr(self.api, '_invalidate_all'): self.api._invalidate_all() def test_ring_packages(self): """ Validate the creation of the rings. """ # our content in the XML files # test content for listonly ie. list command ring_packages = { 'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-1': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-2': 'openSUSE:Factory:Rings:2-TestDVD', 'git': 'openSUSE:Factory:Rings:2-TestDVD', 'wine': 'openSUSE:Factory:Rings:1-MinimalX', } self.assertEqual(ring_packages, self.api.ring_packages_for_links) # test content for real usage ring_packages = { 'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-1': 'openSUSE:Factory:Rings:1-MinimalX', 'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-2': 'openSUSE:Factory:Rings:2-TestDVD', 'git': 'openSUSE:Factory:Rings:2-TestDVD', 'wine': 'openSUSE:Factory:Rings:1-MinimalX', } self.assertEqual(ring_packages, self.api.ring_packages) @unittest.skip("no longer approving non-ring packages") def test_dispatch_open_requests(self): """ Test dispatching and closure of non-ring packages """ # Get rid of open requests self.api.dispatch_open_requests() # Check that we tried to close it self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate']) # Try it again self.api.dispatch_open_requests() # This time there should be nothing to close self.assertEqual(httpretty.last_request().method, 'GET') def test_pseudometa_get_prj(self): """ Test getting project metadata from YAML in project description """ # Try to get data from project that has no metadata data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A') # Should be empty, but contain structure to work with self.assertEqual(data, {'requests': []}) # Add some sample data rq = {'id': '123', 'package': 'test-package'} data['requests'].append(rq) # Save them and read them back self.api.set_prj_pseudometa('openSUSE:Factory:Staging:A', data) test_data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A') # Verify that we got back the same data self.assertEqual(data, test_data) def test_list_projects(self): """ List projects and their content """ # Prepare expected results data = [] for prj in self.obs.staging_project: data.append('openSUSE:Factory:Staging:' + prj) # Compare the results self.assertEqual(data, self.api.get_staging_projects()) def test_open_requests(self): """ Test searching for open requests """ requests = [] # get the open requests requests = self.api.get_open_requests() # Compare the results, we only care now that we got 1 of them not the content self.assertEqual(1, len(requests)) def test_get_package_information(self): """ Test if we get proper project, name and revision from the staging informations """ package_info = { 'dir_srcmd5': '751efeae52d6c99de48164088a33d855', 'project': 'home:Admin', 'rev': '7b98ac01b8071d63a402fa99dc79331c', 'srcmd5': '7b98ac01b8071d63a402fa99dc79331c', 'package': 'wine' } # Compare the results, we only care now that we got 2 of them not the content self.assertEqual( package_info, self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine')) def test_request_id_package_mapping(self): """ Test whether we can get correct id for sr in staging project """ prj = 'openSUSE:Factory:Staging:B' # Get rq num = self.api.get_request_id_for_package(prj, 'wine') self.assertEqual(333, num) # Get package name self.assertEqual('wine', self.api.get_package_for_request_id(prj, num)) def test_rm_from_prj(self): prj = 'openSUSE:Factory:Staging:B' pkg = 'wine' full_name = prj + '/' + pkg # Verify package is there self.assertTrue(full_name in self.obs.links) # Get rq number num = self.api.get_request_id_for_package(prj, pkg) # Delete the package self.api.rm_from_prj(prj, package='wine') # Verify package is not there self.assertTrue(full_name not in self.obs.links) # RQ is gone self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg)) self.assertEqual(None, self.api.get_package_for_request_id(prj, num)) # Verify that review is closed self.assertEqual('accepted', self.obs.requests[str(num)]['review']) self.assertEqual('new', self.obs.requests[str(num)]['request']) def test_rm_from_prj_2(self): # Try the same with request number prj = 'openSUSE:Factory:Staging:B' pkg = 'wine' full_name = prj + '/' + pkg # Get rq number num = self.api.get_request_id_for_package(prj, pkg) # Delete the package self.api.rm_from_prj(prj, request_id=num) # Verify package is not there self.assertTrue(full_name not in self.obs.links) # RQ is gone self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg)) self.assertEqual(None, self.api.get_package_for_request_id(prj, num)) # Verify that review is closed self.assertEqual('accepted', self.obs.requests[str(num)]['review']) self.assertEqual('new', self.obs.requests[str(num)]['request']) def test_add_sr(self): prj = 'openSUSE:Factory:Staging:A' rq = '123' # Running it twice shouldn't change anything for i in range(2): # Add rq to the project self.api.rq_to_prj(rq, prj) # Verify that review is there self.assertEqual('new', self.obs.requests[rq]['review']) self.assertEqual('review', self.obs.requests[rq]['request']) self.assertEqual(self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A'), {'requests': [{'id': 123, 'package': 'gcc', 'author': 'Admin'}]}) def test_create_package_container(self): """Test if the uploaded _meta is correct.""" self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine') self.assertEqual(httpretty.last_request().method, 'PUT') self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/></package>') self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta') self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine', disable_build=True) self.assertEqual(httpretty.last_request().method, 'PUT') self.assertEqual(httpretty.last_request().body, '<package name="wine"><title /><description /><build><disable /></build></package>') self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta') def test_review_handling(self): """Test whether accepting/creating reviews behaves correctly.""" # Add review self.api.add_review('123', by_project='openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview']) # Try to readd, should do anything self.api.add_review('123', by_project='openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'GET') # Accept review self.api.set_review('123', 'openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate']) # Try to accept it again should do anything self.api.set_review('123', 'openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'GET') # But we should be able to reopen it self.api.add_review('123', by_project='openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview']) def test_prj_from_letter(self): # Verify it works self.assertEqual(self.api.prj_from_letter('openSUSE:Factory'), 'openSUSE:Factory') self.assertEqual(self.api.prj_from_letter('A'), 'openSUSE:Factory:Staging:A') def test_frozen_mtime(self): """Test frozen mtime.""" # Testing frozen mtime self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:A') > 8) self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:B') < 1) # U == unfrozen self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:U') > 1000) def test_frozen_enough(self): """Test frozen enough.""" # Testing frozen mtime self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:B'), True) self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:A'), False) # U == unfrozen self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:U'), False) def test_move(self): """Test package movement.""" init_data = self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine') self.api.move_between_project('openSUSE:Factory:Staging:B', 333, 'openSUSE:Factory:Staging:A') test_data = self.api.get_package_information('openSUSE:Factory:Staging:A', 'wine') self.assertEqual(init_data, test_data)
class TestApiCalls(unittest.TestCase): """ Tests for various api calls to ensure we return expected content """ def setUp(self): """ Initialize the configuration """ self.obs = OBS() Config('openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') def tearDown(self): """Clean internal cache""" if hasattr(self.api, '_invalidate_all'): self.api._invalidate_all() def test_ring_packages(self): """ Validate the creation of the rings. """ # our content in the XML files # test content for listonly ie. list command ring_packages = { 'apparmor': 'openSUSE:Factory:Rings:1-MinimalX', 'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-1': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap', 'wine': 'openSUSE:Factory:Rings:1-MinimalX', } self.assertEqual(ring_packages, self.api.ring_packages_for_links) # test content for real usage ring_packages = { 'apparmor': 'openSUSE:Factory:Rings:1-MinimalX', 'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap', 'elem-ring-1': 'openSUSE:Factory:Rings:1-MinimalX', 'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap', 'wine': 'openSUSE:Factory:Rings:1-MinimalX', } self.assertEqual(ring_packages, self.api.ring_packages) @unittest.skip("no longer approving non-ring packages") def test_dispatch_open_requests(self): """ Test dispatching and closure of non-ring packages """ # Get rid of open requests self.api.dispatch_open_requests() # Check that we tried to close it self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate']) # Try it again self.api.dispatch_open_requests() # This time there should be nothing to close self.assertEqual(httpretty.last_request().method, 'GET') def test_pseudometa_get_prj(self): """ Test getting project metadata from YAML in project description """ # Try to get data from project that has no metadata data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A') # Should be empty, but contain structure to work with self.assertEqual(data, {'requests': []}) # Add some sample data rq = {'id': '123', 'package': 'test-package'} data['requests'].append(rq) # Save them and read them back self.api.set_prj_pseudometa('openSUSE:Factory:Staging:A', data) test_data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A') # Verify that we got back the same data self.assertEqual(data, test_data) def test_list_projects(self): """ List projects and their content """ # Prepare expected results data = [] for prj in self.obs.staging_project: data.append('openSUSE:Factory:Staging:' + prj) # Compare the results self.assertEqual(data, self.api.get_staging_projects()) def test_open_requests(self): """ Test searching for open requests """ requests = [] # get the open requests requests = self.api.get_open_requests() # Compare the results, we only care now that we got 1 of them not the content self.assertEqual(1, len(requests)) def test_get_package_information(self): """ Test if we get proper project, name and revision from the staging informations """ package_info = { 'dir_srcmd5': '751efeae52d6c99de48164088a33d855', 'project': 'home:Admin', 'rev': '7b98ac01b8071d63a402fa99dc79331c', 'srcmd5': '7b98ac01b8071d63a402fa99dc79331c', 'package': 'wine' } # Compare the results, we only care now that we got 2 of them not the content self.assertEqual( package_info, self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine')) def test_request_id_package_mapping(self): """ Test whether we can get correct id for sr in staging project """ prj = 'openSUSE:Factory:Staging:B' # Get rq num = self.api.get_request_id_for_package(prj, 'wine') self.assertEqual(333, num) # Get package name self.assertEqual('wine', self.api.get_package_for_request_id(prj, num)) def test_rm_from_prj(self): prj = 'openSUSE:Factory:Staging:B' pkg = 'wine' full_name = prj + '/' + pkg # Verify package is there self.assertTrue(full_name in self.obs.links) # Get rq number num = self.api.get_request_id_for_package(prj, pkg) # Delete the package self.api.rm_from_prj(prj, package='wine') # Verify package is not there self.assertTrue(full_name not in self.obs.links) # RQ is gone self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg)) self.assertEqual(None, self.api.get_package_for_request_id(prj, num)) # Verify that review is closed self.assertEqual('accepted', self.obs.requests[str(num)]['review']) self.assertEqual('new', self.obs.requests[str(num)]['request']) def test_rm_from_prj_2(self): # Try the same with request number prj = 'openSUSE:Factory:Staging:B' pkg = 'wine' full_name = prj + '/' + pkg # Get rq number num = self.api.get_request_id_for_package(prj, pkg) # Delete the package self.api.rm_from_prj(prj, request_id=num) # Verify package is not there self.assertTrue(full_name not in self.obs.links) # RQ is gone self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg)) self.assertEqual(None, self.api.get_package_for_request_id(prj, num)) # Verify that review is closed self.assertEqual('accepted', self.obs.requests[str(num)]['review']) self.assertEqual('new', self.obs.requests[str(num)]['request']) def test_add_sr(self): prj = 'openSUSE:Factory:Staging:A' rq = '123' # Running it twice shouldn't change anything for i in range(2): # Add rq to the project self.api.rq_to_prj(rq, prj) # Verify that review is there self.assertEqual('new', self.obs.requests[rq]['review']) self.assertEqual('review', self.obs.requests[rq]['request']) self.assertEqual(self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A'), {'requests': [{'id': 123, 'package': 'gcc', 'author': 'Admin', 'type': 'submit'}]}) def test_create_package_container(self): """Test if the uploaded _meta is correct.""" self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine') self.assertEqual(httpretty.last_request().method, 'PUT') self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/></package>') self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta') self.api.create_package_container('openSUSE:Factory:Staging:B', 'wine', disable_build=True) self.assertEqual(httpretty.last_request().method, 'PUT') self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/><build><disable/></build></package>') self.assertEqual(httpretty.last_request().path, '/source/openSUSE:Factory:Staging:B/wine/_meta') def test_review_handling(self): """Test whether accepting/creating reviews behaves correctly.""" # Add review self.api.add_review('123', by_project='openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview']) # Try to readd, should do anything self.api.add_review('123', by_project='openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'GET') # Accept review self.api.set_review('123', 'openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate']) # Try to accept it again should do anything self.api.set_review('123', 'openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'GET') # But we should be able to reopen it self.api.add_review('123', by_project='openSUSE:Factory:Staging:A') self.assertEqual(httpretty.last_request().method, 'POST') self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'addreview']) def test_prj_from_letter(self): # Verify it works self.assertEqual(self.api.prj_from_letter('openSUSE:Factory'), 'openSUSE:Factory') self.assertEqual(self.api.prj_from_letter('A'), 'openSUSE:Factory:Staging:A') def test_frozen_mtime(self): """Test frozen mtime.""" # Testing frozen mtime self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:A') > 8) self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:B') < 1) # U == unfrozen self.assertTrue(self.api.days_since_last_freeze('openSUSE:Factory:Staging:U') > 1000) def test_frozen_enough(self): """Test frozen enough.""" # Testing frozen mtime self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:B'), True) self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:A'), False) # U == unfrozen self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:U'), False) def test_move(self): """Test package movement.""" init_data = self.api.get_package_information('openSUSE:Factory:Staging:B', 'wine') self.api.move_between_project('openSUSE:Factory:Staging:B', 333, 'openSUSE:Factory:Staging:A') test_data = self.api.get_package_information('openSUSE:Factory:Staging:A', 'wine') self.assertEqual(init_data, test_data)
class TestApiCalls(unittest.TestCase): """ Tests for various api calls to ensure we return expected content """ def setUp(self): """ Initialize the configuration """ self.obs = OBS() Config("openSUSE:Factory") self.api = StagingAPI(APIURL, "openSUSE:Factory") def tearDown(self): """Clean internal cache""" self.api._invalidate_all() def test_ring_packages(self): """ Validate the creation of the rings. """ # our content in the XML files ring_packages = { "elem-ring-0": "openSUSE:Factory:Rings:0-Bootstrap", "elem-ring-1": "openSUSE:Factory:Rings:1-MinimalX", "elem-ring-2": "openSUSE:Factory:Rings:2-TestDVD", "git": "openSUSE:Factory:Rings:2-TestDVD", "wine": "openSUSE:Factory:Rings:1-MinimalX", } self.assertEqual(ring_packages, self.api.ring_packages) @unittest.skip("no longer approving non-ring packages") def test_dispatch_open_requests(self): """ Test dispatching and closure of non-ring packages """ # Get rid of open requests self.api.dispatch_open_requests() # Check that we tried to close it self.assertEqual(httpretty.last_request().method, "POST") self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"changereviewstate"]) # Try it again self.api.dispatch_open_requests() # This time there should be nothing to close self.assertEqual(httpretty.last_request().method, "GET") def test_pseudometa_get_prj(self): """ Test getting project metadata from YAML in project description """ # Try to get data from project that has no metadata data = self.api.get_prj_pseudometa("openSUSE:Factory:Staging:A") # Should be empty, but contain structure to work with self.assertEqual(data, {"requests": []}) # Add some sample data rq = {"id": "123", "package": "test-package"} data["requests"].append(rq) # Save them and read them back self.api.set_prj_pseudometa("openSUSE:Factory:Staging:A", data) test_data = self.api.get_prj_pseudometa("openSUSE:Factory:Staging:A") # Verify that we got back the same data self.assertEqual(data, test_data) def test_list_projects(self): """ List projects and their content """ # Prepare expected results data = [] for prj in self.obs.staging_project: data.append("openSUSE:Factory:Staging:" + prj) # Compare the results self.assertEqual(data, self.api.get_staging_projects()) def test_open_requests(self): """ Test searching for open requests """ requests = [] # get the open requests requests = self.api.get_open_requests() # Compare the results, we only care now that we got 1 of them not the content self.assertEqual(1, len(requests)) def test_get_package_information(self): """ Test if we get proper project, name and revision from the staging informations """ package_info = { "dir_srcmd5": "751efeae52d6c99de48164088a33d855", "project": "home:Admin", "rev": "7b98ac01b8071d63a402fa99dc79331c", "srcmd5": "7b98ac01b8071d63a402fa99dc79331c", "package": "wine", } # Compare the results, we only care now that we got 2 of them not the content self.assertEqual(package_info, self.api.get_package_information("openSUSE:Factory:Staging:B", "wine")) def test_request_id_package_mapping(self): """ Test whether we can get correct id for sr in staging project """ prj = "openSUSE:Factory:Staging:B" # Get rq num = self.api.get_request_id_for_package(prj, "wine") self.assertEqual(333, num) # Get package name self.assertEqual("wine", self.api.get_package_for_request_id(prj, num)) def test_rm_from_prj(self): prj = "openSUSE:Factory:Staging:B" pkg = "wine" full_name = prj + "/" + pkg # Verify package is there self.assertTrue(full_name in self.obs.links) # Get rq number num = self.api.get_request_id_for_package(prj, pkg) # Delete the package self.api.rm_from_prj(prj, package="wine") # Verify package is not there self.assertTrue(full_name not in self.obs.links) # RQ is gone self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg)) self.assertEqual(None, self.api.get_package_for_request_id(prj, num)) # Verify that review is closed self.assertEqual("accepted", self.obs.requests[str(num)]["review"]) self.assertEqual("new", self.obs.requests[str(num)]["request"]) def test_rm_from_prj_2(self): # Try the same with request number prj = "openSUSE:Factory:Staging:B" pkg = "wine" full_name = prj + "/" + pkg # Get rq number num = self.api.get_request_id_for_package(prj, pkg) # Delete the package self.api.rm_from_prj(prj, request_id=num) # Verify package is not there self.assertTrue(full_name not in self.obs.links) # RQ is gone self.assertEqual(None, self.api.get_request_id_for_package(prj, pkg)) self.assertEqual(None, self.api.get_package_for_request_id(prj, num)) # Verify that review is closed self.assertEqual("accepted", self.obs.requests[str(num)]["review"]) self.assertEqual("new", self.obs.requests[str(num)]["request"]) def test_add_sr(self): prj = "openSUSE:Factory:Staging:A" rq = "123" # Running it twice shouldn't change anything for i in range(2): # Add rq to the project self.api.rq_to_prj(rq, prj) # Verify that review is there self.assertEqual("new", self.obs.requests[rq]["review"]) self.assertEqual("review", self.obs.requests[rq]["request"]) self.assertEqual( self.api.get_prj_pseudometa("openSUSE:Factory:Staging:A"), {"requests": [{"id": 123, "package": "gcc", "author": "Admin"}]}, ) def test_create_package_container(self): """Test if the uploaded _meta is correct.""" self.api.create_package_container("openSUSE:Factory:Staging:B", "wine") self.assertEqual(httpretty.last_request().method, "PUT") self.assertEqual(httpretty.last_request().body, '<package name="wine"><title/><description/></package>') self.assertEqual(httpretty.last_request().path, "/source/openSUSE:Factory:Staging:B/wine/_meta") self.api.create_package_container("openSUSE:Factory:Staging:B", "wine", disable_build=True) self.assertEqual(httpretty.last_request().method, "PUT") self.assertEqual( httpretty.last_request().body, '<package name="wine"><title /><description /><build><disable /></build></package>', ) self.assertEqual(httpretty.last_request().path, "/source/openSUSE:Factory:Staging:B/wine/_meta") def test_review_handling(self): """Test whether accepting/creating reviews behaves correctly.""" # Add review self.api.add_review("123", by_project="openSUSE:Factory:Staging:A") self.assertEqual(httpretty.last_request().method, "POST") self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"addreview"]) # Try to readd, should do anything self.api.add_review("123", by_project="openSUSE:Factory:Staging:A") self.assertEqual(httpretty.last_request().method, "GET") # Accept review self.api.set_review("123", "openSUSE:Factory:Staging:A") self.assertEqual(httpretty.last_request().method, "POST") self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"changereviewstate"]) # Try to accept it again should do anything self.api.set_review("123", "openSUSE:Factory:Staging:A") self.assertEqual(httpretty.last_request().method, "GET") # But we should be able to reopen it self.api.add_review("123", by_project="openSUSE:Factory:Staging:A") self.assertEqual(httpretty.last_request().method, "POST") self.assertEqual(httpretty.last_request().querystring[u"cmd"], [u"addreview"]) def test_prj_from_letter(self): # Verify it works self.assertEqual(self.api.prj_from_letter("openSUSE:Factory"), "openSUSE:Factory") self.assertEqual(self.api.prj_from_letter("A"), "openSUSE:Factory:Staging:A") def test_frozen_mtime(self): """Test frozen mtime.""" # Testing frozen mtime self.assertTrue(self.api.days_since_last_freeze("openSUSE:Factory:Staging:A") > 8) self.assertTrue(self.api.days_since_last_freeze("openSUSE:Factory:Staging:B") < 1) # U == unfrozen self.assertTrue(self.api.days_since_last_freeze("openSUSE:Factory:Staging:U") > 1000) def test_frozen_enough(self): """Test frozen enough.""" # Testing frozen mtime self.assertEqual(self.api.prj_frozen_enough("openSUSE:Factory:Staging:B"), True) self.assertEqual(self.api.prj_frozen_enough("openSUSE:Factory:Staging:A"), False) # U == unfrozen self.assertEqual(self.api.prj_frozen_enough("openSUSE:Factory:Staging:U"), False) def test_move(self): """Test package movement.""" init_data = self.api.get_package_information("openSUSE:Factory:Staging:B", "wine") self.api.move_between_project("openSUSE:Factory:Staging:B", 333, "openSUSE:Factory:Staging:A") test_data = self.api.get_package_information("openSUSE:Factory:Staging:A", "wine") self.assertEqual(init_data, test_data)
class StagingHelper(object): def __init__(self, project): self.project = project self.apiurl = osc.conf.config['apiurl'] self.debug = osc.conf.config['debug'] Config(self.project) self.api = StagingAPI(self.apiurl, self.project) def load_rebuild_data(self, project, package, filename): url = makeurl( self.apiurl, ['source', project, package, '{}?expand=1'.format(filename)]) try: return http_GET(url) except urllib2.HTTPError: return None def save_rebuild_data(self, project, package, filename, content): url = makeurl(self.apiurl, ['source', project, package, filename]) http_PUT(url + '?comment=support+package+rebuild+update', data=content) def rebuild_project(self, project): query = {'cmd': 'rebuild'} url = makeurl(self.apiurl, ['build', project], query=query) http_POST(url) def get_source_packages(self, project): """Return the list of packages in a project.""" query = {'expand': 1} root = ET.parse( http_GET(makeurl(self.apiurl, ['source', project], query=query))).getroot() packages = [i.get('name') for i in root.findall('entry')] return packages def get_support_package_list(self, project, repository): f = osc.core.get_buildconfig(self.apiurl, project, repository).splitlines() pkg_list = [] for line in f: if re.match('Preinstall', line) or re.match('VMinstall', line): content = line.split(':') variables = [x.strip() for x in content[1].split(' ')] for var in variables: if var != '' and var not in pkg_list: if var.startswith('!') and var[1:] in pkg_list: pkg_list.remove(var[1:]) else: pkg_list.append(var) return pkg_list def get_project_binarylist(self, project, repository, arch): query = {'view': 'binarylist', 'repository': repository, 'arch': arch} root = ET.parse( http_GET( makeurl(self.apiurl, ['build', project, '_result'], query=query))).getroot() return root def get_package_buildinfo(self, project, repository, arch, package): url = makeurl( self.apiurl, ['build', project, repository, arch, package, '_buildinfo']) root = ET.parse(http_GET(url)).getroot() return root def get_buildinfo_version(self, project, package): buildinfo = self.get_package_buildinfo(project, 'standard', 'x86_64', package) versrel = buildinfo.find('versrel') version = versrel.split('-')[0] return version def process_project_binarylist(self, project, repository, arch): prj_binarylist = self.get_project_binarylist(project, repository, arch) files = {} for package in prj_binarylist.findall('./result/binarylist'): for binary in package.findall('binary'): result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', binary.attrib['filename']) if not result: continue bname = result.group(1) if bname.endswith('-debuginfo') or bname.endswith( '-debuginfo-32bit'): continue if bname.endswith('-debugsource'): continue if bname.startswith('::import::'): continue if result.group(4) == 'src': continue files[bname] = package.attrib['package'] return files def crawl(self): """Main method""" rebuild_data = self.load_rebuild_data(self.project + ':Staging', 'dashboard', 'support_pkg_rebuild') if rebuild_data is None: print "There is no support_pkg_rebuild file!" return logging.info('Gathering support package list from %s' % self.project) support_pkgs = self.get_support_package_list(self.project, 'standard') files = self.process_project_binarylist(self.project, 'standard', 'x86_64') staging_projects = [ "%s:%s" % (self.api.cstaging, p) for p in self.api.get_staging_projects_short() ] cand_sources = defaultdict(list) for stg in staging_projects: prj_meta = self.api.get_prj_pseudometa(stg) prj_staged_packages = [ req['package'] for req in prj_meta['requests'] ] for pkg in support_pkgs: if files.get(pkg) and files.get(pkg) in prj_staged_packages: if files.get(pkg) not in cand_sources[stg]: cand_sources[stg].append(files.get(pkg)) tree = ET.parse(rebuild_data) root = tree.getroot() logging.info('Checking rebuild data...') update_suppkgs = False for stg in root.findall('staging'): rebuild = stg.find('rebuild').text suppkg_list = stg.find('supportpkg').text need_rebuild = False suppkgs = [] if suppkg_list: suppkgs = suppkg_list.split(',') stgname = stg.get('name') if len(cand_sources[stgname]) and rebuild == 'unknown': need_rebuild = True stg.find('rebuild').text = 'needed' update_suppkgs = True new_suppkg_list = ','.join(cand_sources[stgname]) stg.find('supportpkg').text = new_suppkg_list elif len(cand_sources[stgname]) and rebuild != 'unknown': if rebuild == 'unneeded': for cand in cand_sources[stgname]: if cand not in suppkgs: need_rebuild = True stg.find('rebuild').text = 'needed' break update_suppkgs = True new_suppkg_list = ','.join(cand_sources[stgname]) stg.find('supportpkg').text = new_suppkg_list elif not len(cand_sources[stgname]): stg.find('rebuild').text = 'unneeded' if need_rebuild and not self.api.is_repo_dirty( stgname, 'standard'): logging.info('Rebuild %s' % stgname) self.rebuild_project(stgname) stg.find('rebuild').text = 'unneeded' if update_suppkgs: logging.info('Updating support pkg list...') logging.debug(ET.tostring(root)) self.save_rebuild_data(self.project + ':Staging', 'dashboard', 'support_pkg_rebuild', ET.tostring(root))
class StagingHelper(object): def __init__(self, project): self.project = project self.apiurl = osc.conf.config['apiurl'] Config(self.apiurl, self.project) self.api = StagingAPI(self.apiurl, self.project) def get_support_package_list(self, project, repository): f = osc.core.get_buildconfig(self.apiurl, project, repository).splitlines() pkg_list = [] for line in f: if re.match('Preinstall', line) or re.match('VM[Ii]nstall', line) or re.match('Support', line): content = line.split(':') variables = [x.strip() for x in content[1].split(' ')] for var in variables: if var != '' and var not in pkg_list: if var.startswith('!') and var[1:] in pkg_list: pkg_list.remove(var[1:]) else: pkg_list.append(var) return pkg_list def get_project_binarylist(self, project, repository, arch): query = {'view': 'binaryversions'} root = ET.parse(http_GET(makeurl(self.apiurl, ['build', project, repository, arch], query=query))).getroot() return root def process_project_binarylist(self, project, repository, arch): prj_binarylist = self.get_project_binarylist(project, repository, arch) files = {} for package in prj_binarylist.findall('./binaryversionlist'): for binary in package.findall('binary'): result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', binary.attrib['name']) if not result: continue bname = result.group(1) if bname.endswith('-debuginfo') or bname.endswith('-debuginfo-32bit'): continue if bname.endswith('-debugsource'): continue if bname.startswith('::import::'): continue if result.group(4) == 'src': continue files[bname] = package.attrib['package'].split(':', 1)[0] return files def check_multiple_specs(self, project, packages): expanded_packages = [] for pkg in packages: query = {'expand': 1} url = makeurl(self.apiurl, ['source', project, pkg], query=query) try: root = ET.parse(http_GET(url)).getroot() except HTTPError as e: if e.code == 404: continue raise for en in root.findall('entry'): if en.attrib['name'].endswith('.spec'): expanded_packages.append(en.attrib['name'][:-5]) return expanded_packages def crawl(self): """Main method""" rebuild_data = self.api.pseudometa_file_load('support_pkg_rebuild') if rebuild_data is None: print("There is no support_pkg_rebuild file!") return logging.info('Gathering support package list from %s' % self.project) support_pkgs = self.get_support_package_list(self.project, 'standard') files = self.process_project_binarylist(self.project, 'standard', 'x86_64') staging_projects = ["%s:%s"%(self.api.cstaging, p) for p in self.api.get_staging_projects_short()] cand_sources = defaultdict(list) for stg in staging_projects: prj_meta = self.api.get_prj_pseudometa(stg) prj_staged_packages = [req['package'] for req in prj_meta['requests']] prj_expanded_packages = self.check_multiple_specs(self.project, prj_staged_packages) for pkg in support_pkgs: if files.get(pkg) and files.get(pkg) in prj_expanded_packages: if files.get(pkg) not in cand_sources[stg]: cand_sources[stg].append(files.get(pkg)) root = ET.fromstring(rebuild_data) logging.info('Checking rebuild data...') for stg in root.findall('staging'): rebuild = stg.find('rebuild').text suppkg_list = stg.find('supportpkg').text need_rebuild = False suppkgs = [] if suppkg_list: suppkgs = suppkg_list.split(',') stgname = stg.get('name') if len(cand_sources[stgname]) and rebuild == 'unknown': need_rebuild = True stg.find('rebuild').text = 'needed' new_suppkg_list = ','.join(cand_sources[stgname]) stg.find('supportpkg').text = new_suppkg_list elif len(cand_sources[stgname]) and rebuild != 'unknown': for cand in cand_sources[stgname]: if cand not in suppkgs: need_rebuild = True stg.find('rebuild').text = 'needed' break new_suppkg_list = ','.join(cand_sources[stgname]) stg.find('supportpkg').text = new_suppkg_list elif not len(cand_sources[stgname]): stg.find('rebuild').text = 'unneeded' stg.find('supportpkg').text = '' if stg.find('rebuild').text == 'needed': need_rebuild = True if need_rebuild and not self.api.is_repo_dirty(stgname, 'standard'): logging.info('Rebuild %s' % stgname) osc.core.rebuild(self.apiurl, stgname, None, None, None) stg.find('rebuild').text = 'unneeded' logging.info('Updating support pkg list...') rebuild_data_updated = ET.tostring(root) logging.debug(rebuild_data_updated) if rebuild_data_updated != rebuild_data: self.api.pseudometa_file_save( 'support_pkg_rebuild', rebuild_data_updated, 'support package rebuild')