def _get_available_links(self, website):
list_urls = []
html = self._get_response_website(website)
search_websites_response = BeautifulSoup(html, 'lxml')\
.find_all('a', href=True)
if search_websites_response:
for url_website in search_websites_response:
url_website = url_website['href']
if is_valid_url(url_website):
list_urls.append(url_website)
return list_urls
def test_is_valid_url_withou_string_parm(self):
with self.assertRaisesRegexp(ValueError, "Url need be string"):
is_valid_url(0)
def test_is_valid_url(self, value, expected):
self.assertEquals(is_valid_url(value), expected)
def _put_website_links_on_quee(self, list_urls):
for url_website in list_urls:
if is_valid_url(url_website):
self._save_website(url_website)
def validate_url(self, key, value):
if not is_valid_url(value):
raise ValueError("Url need be a valid url")
return value