Example #1
0
def _get_blocks(url: str) -> Sequence[BeautifulSoup]:
    """
        Grab all blocks containing news titles and links
        from URL
    """
    result = []
    content = _get_content(url)
    if content:
        try:
            page = html.fromstring(content)
            result = page.find_class('subtitle')
        except OSError:
            page = BeautifulSoup(content, 'lxml')
            result = page.findAll('div', {'class': 'subtitle'})
    return result
 def test_get_content(self):
     content = _get_content(self.url)
     self.assertEqual(len(content), 48233)
 def test_get_content_bad_link(self):
     content = _get_content('htt://googl.fa')
     self.assertEqual(content, '')
 def test_get_content(self):
     content = _get_content(self.url)
     self.assertEqual(len(content), 48233)
 def test_get_content_bad_link(self):
     content = _get_content('htt://googl.fa')
     self.assertEqual(content, '')
 def test_get_content_bad_link(self):
     content = _get_content("htt://googl.fa")
     self.assertEqual(content, "")