def test_rendering(self): default_repo = self.factory.makeGitRepository( owner=self.owner, target=self.target, name="foo") self.factory.makeGitRefs( default_repo, paths=["refs/heads/master", "refs/heads/bug-1234"]) other_repo = self.factory.makeGitRepository( owner=self.owner, target=self.target, name="bar") self.factory.makeGitRefs(other_repo, paths=["refs/heads/bug-2468"]) with admin_logged_in(): getUtility(IGitRepositorySet).setDefaultRepositoryForOwner( owner=self.owner, target=self.target, repository=default_repo, user=self.owner) view = create_initialized_view(self.owner_target, '+git') self.assertEqual(default_repo, view.default_git_repository) content = view() soup = BeautifulSoup(content) # Clone instructions for the default repo are present. self.assertEqual( 'https://git.launchpad.dev/~dev/%s' % self.target_path, soup.find(attrs={'class': 'https-url'}).find(text=True)) self.assertEqual( 'https://git.launchpad.dev/~dev/%s' % self.target_path, soup.find(text='Browse the code').parent['href']) # The default repo's branches are shown. table = soup.find( 'div', id='default-repository-branches').find('table') self.assertContentEqual( ['master', 'bug-1234'], [link.find(text=True) for link in table.findAll('a')]) self.assertEndsWith( table.find(text="bug-1234").parent['href'], "/~dev/%s/+git/foo/+ref/bug-1234" % self.target_path) # Other repos are listed. table = soup.find( 'div', id='gitrepositories-table-listing').find('table') self.assertContentEqual( ['lp:~dev/%s' % self.target_path, 'lp:~dev/%s/+git/bar' % self.target_path], [link.find(text=True) for link in table.findAll('a')]) self.assertEndsWith( table.find( text="lp:~dev/%s/+git/bar" % self.target_path).parent['href'], "/~dev/%s/+git/bar" % self.target_path) # But not their branches. self.assertNotIn('bug-2468', content)
def test_view_contents(self): # Distributions are rendered with the correct text. url = canonical_url(self.pillar, view_name='+sharing') browser = setupBrowserForUser(user=self.driver) browser.open(url) soup = BeautifulSoup(browser.contents) commercial_text = soup.find('p', {'id': 'commercial-project-text'}) non_commercial_text = soup.find( 'p', {'id': 'non-commercial-project-text'}) self.assertIsNone(commercial_text) self.assertIsNone(non_commercial_text)
def test_view_contents_commercial_project(self): # Commercial projects are rendered with the correct text. self.factory.makeCommercialSubscription(self.pillar) url = canonical_url(self.pillar, view_name='+sharing') browser = setupBrowserForUser(user=self.driver) browser.open(url) soup = BeautifulSoup(browser.contents) commercial_text = soup.find('p', {'id': 'commercial-project-text'}) non_commercial_text = soup.find( 'p', {'id': 'non-commercial-project-text'}) self.assertIsNotNone(commercial_text) self.assertIsNone(non_commercial_text)
def test_filebug_view_renders_info_type_widget(self): # The info type widget is rendered for bug supervisor roles. product = self.factory.makeProduct(official_malone=True) with person_logged_in(product.owner): view = create_initialized_view(product, '+filebug', principal=product.owner) html = view.render() soup = BeautifulSoup(html) self.assertIsNone( soup.find('input', attrs={'name': 'field.security_related'})) self.assertIsNotNone( soup.find('input', attrs={'name': 'field.information_type'}))
def test_filebug_view_renders_security_related(self): # The security_related checkbox is rendered for non bug supervisors. product = self.factory.makeProduct(official_malone=True) anyone = self.factory.makePerson() with person_logged_in(anyone): view = create_initialized_view(product, '+filebug', principal=anyone) html = view.render() soup = BeautifulSoup(html) self.assertIsNotNone( soup.find('input', attrs={'name': 'field.security_related'})) self.assertIsNone( soup.find('input', attrs={'name': 'field.information_type'}))
def test_rendering(self): some_repo = self.factory.makeGitRepository( owner=self.owner, target=self.target, name="foo") self.factory.makeGitRefs( some_repo, paths=["refs/heads/master", "refs/heads/bug-1234"]) other_repo = self.factory.makeGitRepository( owner=self.owner, target=self.target, name="bar") self.factory.makeGitRefs(other_repo, paths=["refs/heads/bug-2468"]) view = create_initialized_view(self.context, '+git') self.assertIs(None, view.default_git_repository) content = view() soup = BeautifulSoup(content) # No details about the default repo are shown, as a person # without a target doesn't have a default repo self.assertNotIn('Branches', content) self.assertNotIn('Browse the code', content) self.assertNotIn('git clone', content) self.assertNotIn('bug-1234', content) # All owned repos are listed. table = soup.find( 'div', id='gitrepositories-table-listing').find('table') self.assertContentEqual( [some_repo.git_identity, other_repo.git_identity], [link.find(text=True) for link in table.findAll('a')])
def print_upstream_linking_form(browser): """Print the upstream linking form found via +choose-affected-product. The resulting output will look something like: (*) A checked option [A related text field] ( ) An unchecked option """ soup = BeautifulSoup(browser.contents) link_upstream_how_radio_control = browser.getControl( name='field.link_upstream_how') link_upstream_how_buttons = soup.findAll( 'input', {'name': 'field.link_upstream_how'}) wrapper = textwrap.TextWrapper(width=65, subsequent_indent=' ') for button in link_upstream_how_buttons: # Print the radio button. label = button.findParent('label') if label is None: label = soup.find('label', {'for': button['id']}) if button.get('value') in link_upstream_how_radio_control.value: print wrapper.fill('(*) %s' % extract_text(label)) else: print wrapper.fill('( ) %s' % extract_text(label)) # Print related text field, if found. Assumes that the text # field is in the same table row as the radio button. text_field = button.findParent('tr').find('input', {'type': 'text'}) if text_field is not None: text_control = browser.getControl(name=text_field.get('name')) print ' [%s]' % text_control.value.ljust(10)
def test_does_display_child_diff(self): # If the child's latest published version is not the same as the base # version, we display two links to two diffs. changelog_lfa = self.factory.makeChangelog('foo', ['0.1-1derived1', '0.1-1']) parent_changelog_lfa = self.factory.makeChangelog( 'foo', ['0.1-2', '0.1-1']) transaction.commit() # Yay, librarian. ds_diff = self.factory.makeDistroSeriesDifference( versions={ 'derived': '0.1-1derived1', 'parent': '0.1-2', }, changelogs={ 'derived': changelog_lfa, 'parent': parent_changelog_lfa }) self.assertEqual('0.1-1', ds_diff.base_version) with person_logged_in(self.factory.makePerson()): view = create_initialized_view(ds_diff, '+listing-distroseries-extra') soup = BeautifulSoup(view()) tags = soup.find('ul', 'package-diff-status').findAll('span') self.assertEqual(2, len(tags))
def test_product_with_edit_permission(self): login_person(self.product.owner) self.view = create_initialized_view( self.product, '+questions', principal=self.product.owner) content = BeautifulSoup(self.view()) self.assertCommonPageElements(content) self.assertTrue( content.find(True, id='configure-support') is not None)
def test_sharing_menu(self): url = canonical_url(self.pillar) sharing_url = canonical_url(self.pillar, view_name='+sharing') browser = setupBrowserForUser(user=self.driver) browser.open(url) soup = BeautifulSoup(browser.contents) sharing_menu = soup.find('a', {'href': sharing_url}) self.assertIsNotNone(sharing_menu)
def get_text_values(self, bug): """Returns the text of status and substatus for the given bug. This is done by downloading the HTML bug page and scraping it. """ url = '%s%s' % (self.base_url, bug['id']) page = self.fetch(url).read() soup = BeautifulSoup(page) return tuple( node.string for node in soup.find('th', text='Status').findNext('td').findAll('span'))
def test_filebug_information_type_public_policy(self): # The vocabulary for information_type when filing a bug is created # correctly for non commercial projects. product = self.factory.makeProduct(official_malone=True) with person_logged_in(product.owner): view = create_initialized_view(product, '+filebug', principal=product.owner) html = view.render() soup = BeautifulSoup(html) self.assertIsNone(soup.find('label', text="Proprietary"))
def test_filebug_information_type_vocabulary(self): # The vocabulary for information_type when filing a bug is created # correctly. product = self.factory.makeProduct(official_malone=True) with person_logged_in(product.owner): view = create_initialized_view(product, '+filebug', principal=product.owner) html = view.render() soup = BeautifulSoup(html) for info_type in product.getAllowedBugInformationTypes(): self.assertIsNotNone(soup.find('label', text=info_type.title))
def test_filebug_information_type_proprietary_policy(self): # The vocabulary for information_type when filing a bug is created # correctly for a project with a proprietary sharing policy. product = self.factory.makeProduct(official_malone=True) self.factory.makeCommercialSubscription(product=product) with person_logged_in(product.owner): product.setBugSharingPolicy(BugSharingPolicy.PROPRIETARY) view = create_initialized_view(product, '+filebug', principal=product.owner) html = view.render() soup = BeautifulSoup(html) self.assertIsNotNone(soup.find('label', text="Proprietary"))
def test_anonymous_view_specification_with_private_subscriber(self): # A specification with a private subscriber is not rendered for anon. private_subscriber = self.factory.makeTeam( name="privateteam", visibility=PersonVisibility.PRIVATE) spec = self.factory.makeSpecification() with person_logged_in(spec.owner): spec.subscribe(private_subscriber, spec.owner) # Viewing the specification doesn't display private subscriber. url = canonical_url(spec, rootsite='blueprints') browser = self._getBrowser() browser.open(url) soup = BeautifulSoup(browser.contents) self.assertIsNone( soup.find('div', attrs={'id': 'subscriber-privateteam'}))
def test_view_specification_with_private_subscriber(self): # A specification with a private subscriber is rendered. private_subscriber = self.factory.makeTeam( name="privateteam", visibility=PersonVisibility.PRIVATE) spec = self.factory.makeSpecification() with person_logged_in(spec.owner): spec.subscribe(private_subscriber, spec.owner) # Ensure the specification subscriber is rendered. url = canonical_url(spec, rootsite='blueprints') user = self.factory.makePerson() browser = self._getBrowser(user) browser.open(url) soup = BeautifulSoup(browser.contents) subscriber_portlet = soup.find('div', attrs={'id': 'subscribers'}) self.assertIsNotNone( subscriber_portlet.find('a', text='Privateteam'))
def _extractErrorMessage(self, page_data): """Extract an error message from a SourceForge page and return it.""" soup = BeautifulSoup(page_data) error_frame = soup.find(attrs={'class': 'error'}) if not error_frame: return None # We find the error message by going via the somewhat shakey # method of looking for the only paragraph inside the # error_frame div. error_message = error_frame.find(name='p') if error_message: # Strip out any leading or trailing whitespace and return the # error message. return error_message.string.strip() else: # We know there was an error, but we can't tell what it was. return 'Unspecified error.'
def parse_relationship_section(content): """Parser package relationship section. See package-relationship-pages.txt and related. """ soup = BeautifulSoup(content) section = soup.find('ul') whitespace_re = re.compile('\s+') if section is None: print 'EMPTY SECTION' return for li in section.findAll('li'): if li.a: link = li.a content = whitespace_re.sub(' ', link.string.strip()) url = link['href'] print 'LINK: "%s" -> %s' % (content, url) else: content = whitespace_re.sub(' ', li.string.strip()) print 'TEXT: "%s"' % content
def test_copes_with_no_default(self): self.factory.makeGitRepository( owner=self.owner, target=self.target, name="foo") view = create_initialized_view(self.owner_target, '+git') self.assertIs(None, view.default_git_repository) content = view() soup = BeautifulSoup(content) # No details about the non-existent default repo are shown. # XXX: This should show instructions to create one. self.assertNotIn('Branches', content) self.assertNotIn('Browse the code', content) self.assertNotIn('git clone', content) # Other repos are listed. table = soup.find( 'div', id='gitrepositories-table-listing').find('table') self.assertContentEqual( ['lp:~dev/%s/+git/foo' % self.target_path], [link.find(text=True) for link in table.findAll('a')])
def parsePage(self, page_text): """Builds self.product using HTML content in page_text""" soup = BeautifulSoup(page_text) if soup is None: return None # Load products into a list since Bugzilla references them # by index number products = [] for product in soup.find(name='select', onchange="doOnSelectProduct(2);").contents: if product.string != "\n": products.append({ 'name': product.string, 'components': {}, 'versions': None, }) for script_text in soup.findAll(name="script"): if script_text is None or script_text.string is None: continue for line in script_text.string.split(";"): m = self.re_cpts.search(line) if m: num = int(m.group(1)) products[num]['components'] = dictFromCSV(m.group(2)) m = self.re_vers.search(line) if m: num = int(m.group(1)) products[num]['versions'] = dictFromCSV(m.group(2)) # Re-map list into dict for easier lookups for product in products: product_name = product['name'] self.products[product_name] = product return True
def getRobotsDirective(self): contents = self.getRenderedContents() soup = BeautifulSoup(contents) return soup.find('meta', attrs={'name': 'robots'})
def _configure_project(self, usage): self.naked_product.blueprints_usage = usage view = create_initialized_view(self.product, '+specs') soup = BeautifulSoup(view()) robots = soup.find('meta', attrs={'name': 'robots'}) return soup, robots
def test_rendering(self): main_repo = self.factory.makeGitRepository( owner=self.owner, target=self.target, name="foo") self.factory.makeGitRefs( main_repo, paths=[ "refs/heads/master", "refs/heads/1.0", "refs/heads/with#hash", "refs/heads/\N{SNOWMAN}", "refs/tags/1.1", ]) other_repo = self.factory.makeGitRepository( owner=self.factory.makePerson(name="contributor"), target=self.target, name="foo") self.factory.makeGitRefs(other_repo, paths=["refs/heads/bug-1234"]) self.factory.makeGitRepository( owner=self.factory.makePerson(name="random"), target=self.target, name="bar") with admin_logged_in(): getUtility(IGitRepositorySet).setDefaultRepository( target=self.target, repository=main_repo) getUtility(IGitRepositorySet).setDefaultRepositoryForOwner( owner=other_repo.owner, target=self.target, repository=other_repo, user=other_repo.owner) view = create_initialized_view(self.target, '+git') self.assertEqual(main_repo, view.default_git_repository) content = view() soup = BeautifulSoup(content) # Clone instructions for the default repo are present. self.assertEqual( 'https://git.launchpad.dev/%s' % self.target_path, soup.find(attrs={'class': 'https-url'}).find(text=True)) self.assertEqual( 'https://git.launchpad.dev/%s' % self.target_path, soup.find(text='Browse the code').parent['href']) # The default repo's branches are shown, but not its tags. table = soup.find( 'div', id='default-repository-branches').find('table') self.assertContentEqual( ['1.0', 'master', 'with#hash', '\N{SNOWMAN}'], [link.find(text=True) for link in table.findAll('a')]) self.assertEndsWith( table.find(text="1.0").parent['href'], "/~foowner/%s/+git/foo/+ref/1.0" % self.target_path) self.assertEndsWith( table.find(text="with#hash").parent['href'], "/~foowner/%s/+git/foo/+ref/with%%23hash" % self.target_path) self.assertEndsWith( table.find(text="\N{SNOWMAN}").parent['href'], "/~foowner/%s/+git/foo/+ref/%%E2%%98%%83" % self.target_path) # Other repos are listed. table = soup.find( 'div', id='gitrepositories-table-listing').find('table') self.assertContentEqual( ['lp:%s' % self.target_path, 'lp:~random/%s/+git/bar' % self.target_path, 'lp:~contributor/%s' % self.target_path], [link.find(text=True) for link in table.findAll('a')]) self.assertEndsWith( table.find( text="lp:~contributor/%s" % self.target_path).parent['href'], "/~contributor/%s/+git/foo" % self.target_path) # But not their branches. self.assertNotIn('bug-1234', content)
def initializeRemoteBugDB(self, bug_ids): """See `ExternalBugTracker`. We override this method because SourceForge does not provide a nice way for us to export bug statuses en masse. Instead, we resort to screen-scraping on a per-bug basis. Therefore the usual choice of batch vs. single export does not apply here and we only perform single exports. """ self.bugs = {} for bug_id in bug_ids: query_url = self.export_url % bug_id page_data = self._getPage(query_url).content soup = BeautifulSoup(page_data) status_tag = soup.find(text=re.compile('Status:')) status = None private = False if status_tag: # We can extract the status by finding the grandparent tag. # Happily, BeautifulSoup will turn the contents of this tag # into a newline-delimited list from which we can then # extract the requisite data. status_row = status_tag.findParent().findParent() status = status_row.contents[-1] status = status.strip() else: error_message = self._extractErrorMessage(page_data) # If the error message suggests that the bug is private, # set the bug's private field to True. # XXX 2008-05-01 gmb bug=225354: # We should know more about possible errors and deal # with them accordingly. if error_message and 'private' in error_message.lower(): private = True else: # If we can't find a status line in the output from # SourceForge there's little point in continuing. raise UnparsableBugData( 'Remote bug %s does not define a status.' % bug_id) # We need to do the same for Resolution, though if we can't # find it it's not critical. resolution_tag = soup.find(text=re.compile('Resolution:')) if resolution_tag: resolution_row = resolution_tag.findParent().findParent() resolution = resolution_row.contents[-1] resolution = resolution.strip() else: resolution = None # We save the group_id and atid parameters from the # query_url. They'll be returned by getRemoteProduct(). query_dict = {} bugtracker_link = soup.find('a', text='Bugs') if bugtracker_link: href = bugtracker_link.findParent()['href'] # We need to replace encoded ampersands in the URL since # SourceForge occasionally encodes them. href = href.replace('&', '&') schema, host, path, query, fragment = urlsplit(href) query_bits = query.split('&') for bit in query_bits: key, value = urllib.splitvalue(bit) query_dict[key] = value try: atid = int(query_dict.get('atid', None)) group_id = int(query_dict.get('group_id', None)) except ValueError: atid = None group_id = None else: group_id = None atid = None self.bugs[int(bug_id)] = { 'id': int(bug_id), 'private': private, 'status': status, 'resolution': resolution, 'group_id': group_id, 'atid': atid, }
def test_product_with_packaging_elements(self): self.linkPackage(self.product, 'cow') content = BeautifulSoup(self.view()) self.assertCommonPageElements(content) self.assertTrue(content.find(True, id='ubuntu-support') is not None)
def old_link(self): href = canonical_url(self.target, rootsite=self.rootsite, view_name='+subscribe') soup = BeautifulSoup(self.contents) return soup.find('a', href=href)
def getRemoteProductFromSourceForge(self, sf_project): """Return the remote product of a SourceForge project. :return: The group_id and atid of the SourceForge project's bug tracker as an ampersand-separated string in the form 'group_id&atid'. """ # First, fetch the project page. try: soup = BeautifulSoup(self._getPage("projects/%s" % sf_project)) except requests.HTTPError as error: self.logger.error("Error fetching project %s: %s" % (sf_project, error)) return None # Find the Tracker link and fetch that. tracker_link = soup.find('a', text='Tracker') if tracker_link is None: self.logger.error("No tracker link for project '%s'" % sf_project) return None tracker_url = tracker_link.findParent()['href'] # Clean any leading '/' from tracker_url so that urlappend # doesn't choke on it. tracker_url = tracker_url.lstrip('/') try: soup = BeautifulSoup(self._getPage(tracker_url)) except requests.HTTPError as error: self.logger.error("Error fetching project %s: %s" % (sf_project, error)) return None # Extract the group_id and atid from the bug tracker link. bugtracker_link = soup.find('a', text='Bugs') if bugtracker_link is None: self.logger.error("No bug tracker link for project '%s'" % sf_project) return None bugtracker_url = bugtracker_link.findParent()['href'] # We need to replace encoded ampersands in the URL since # SourceForge usually encodes them. bugtracker_url = bugtracker_url.replace('&', '&') schema, host, path, query, fragment = urlsplit(bugtracker_url) query_dict = {} query_bits = query.split('&') for bit in query_bits: key, value = urllib.splitvalue(bit) query_dict[key] = value try: atid = int(query_dict.get('atid', None)) group_id = int(query_dict.get('group_id', None)) except ValueError: # If anything goes wrong when int()ing the IDs, just return # None. return None return u'%s&%s' % (group_id, atid)